Merge pull request #3805 from freqtrade/new_release

New release 2020.9

.devcontainer/Dockerfile

@@ -0,0 +1,18 @@
+FROM freqtradeorg/freqtrade:develop
+
+# Install dependencies
+COPY requirements-dev.txt /freqtrade/
+RUN apt-get update \
+    && apt-get -y install git sudo vim \
+    && apt-get clean \
+    && pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \
+    && useradd -u 1000 -U -m ftuser \
+    && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \
+    && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \
+    && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \
+    && chown ftuser: -R /home/ftuser/
+
+USER ftuser
+
+# Empty the ENTRYPOINT to allow all commands
+ENTRYPOINT []

.devcontainer/devcontainer.json

@@ -0,0 +1,44 @@
+{
+    "name": "freqtrade Develop",
+
+    "dockerComposeFile": [
+        "docker-compose.yml"
+    ],
+
+    "service": "ft_vscode",
+
+    "workspaceFolder": "/freqtrade/",
+
+    "settings": {
+        "terminal.integrated.shell.linux": "/bin/bash",
+        "editor.insertSpaces": true,
+        "files.trimTrailingWhitespace": true,
+        "[markdown]": {
+            "files.trimTrailingWhitespace": false,
+        },
+        "python.pythonPath": "/usr/local/bin/python",
+    },
+
+    // Add the IDs of extensions you want installed when the container is created.
+    "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "davidanson.vscode-markdownlint",
+        "ms-azuretools.vscode-docker",
+    ],
+
+    // Use 'forwardPorts' to make a list of ports inside the container available locally.
+    // "forwardPorts": [],
+
+    // Uncomment the next line if you want start specific services in your Docker Compose config.
+    // "runServices": [],
+
+    // Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+    // "shutdownAction": "none",
+
+    // Uncomment the next line to run commands after the container is created - for example installing curl.
+    // "postCreateCommand": "sudo apt-get update && apt-get install -y git",
+
+    // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
+    "remoteUser": "ftuser"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,24 @@
+---
+version: '3'
+services:
+  ft_vscode:
+    build:
+      context: ..
+      dockerfile: ".devcontainer/Dockerfile"
+    volumes:
+      # Allow git usage within container
+      - "/home/${USER}/.ssh:/home/ftuser/.ssh:ro"
+      - "/home/${USER}/.gitconfig:/home/ftuser/.gitconfig:ro"
+      - ..:/freqtrade:cached
+      # Persist bash-history
+      - freqtrade-vscode-server:/home/ftuser/.vscode-server
+      - freqtrade-bashhistory:/home/ftuser/commandhistory
+    # Expose API port
+    ports:
+      - "127.0.0.1:8080:8080"
+    command: /bin/sh -c "while sleep 1000; do :; done"
+
+
+volumes:
+  freqtrade-vscode-server:
+  freqtrade-bashhistory:

.dockerignore

@@ -13,3 +13,4 @@ CONTRIBUTING.md
 MANIFEST.in
 README.md
 freqtrade.service
+user_data

.github/workflows/ci.yml

@@ -4,8 +4,8 @@ on:
   push:
     branches:
       - master
+      - stable
       - develop
-      - github_actions_tests
     tags:
   release:
     types: [published]
@@ -88,7 +88,7 @@ jobs:
       run: |
         cp config.json.example config.json
         freqtrade create-userdir --userdir user_data
-        freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt
+        freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --print-all
 
     - name: Flake8
       run: |
@@ -150,7 +150,7 @@ jobs:
       run: |
         cp config.json.example config.json
         freqtrade create-userdir --userdir user_data
-        freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt
+        freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --print-all
 
     - name: Flake8
       run: |
@@ -194,7 +194,7 @@ jobs:
     steps:
     - name: Cleanup previous runs on this branch
       uses: rokroskar/workflow-run-cleanup-action@v0.2.2
-      if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.repository == 'freqtrade/freqtrade'"
+      if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'"
       env:
         GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 
@@ -226,7 +226,7 @@ jobs:
 
     - name: Extract branch name
       shell: bash
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})"
       id: extract_branch
 
     - name: Build distribution
@@ -236,7 +236,7 @@ jobs:
 
     - name: Publish to PyPI (Test)
       uses: pypa/gh-action-pypi-publish@master
-      if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release')
+      if: (steps.extract_branch.outputs.branch == 'stable' || github.event_name == 'release')
       with:
         user: __token__
         password: ${{ secrets.pypi_test_password }}
@@ -244,7 +244,7 @@ jobs:
 
     - name: Publish to PyPI
       uses: pypa/gh-action-pypi-publish@master
-      if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release')
+      if: (steps.extract_branch.outputs.branch == 'stable' || github.event_name == 'release')
       with:
         user: __token__
         password: ${{ secrets.pypi_password }}

.github/workflows/docker_update_readme.yml

@@ -2,7 +2,7 @@ name: Update Docker Hub Description
 on:
   push:
     branches:
-    - master
+    - stable
 
 jobs:
   dockerHubDescription:

CONTRIBUTING.md

@@ -8,8 +8,9 @@ Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/
 
 Few pointers for contributions:
 
-- Create your PR against the `develop` branch, not `master`.
-- New features need to contain unit tests and must be PEP8 conformant (max-line-length = 100).
+- Create your PR against the `develop` branch, not `stable`.
+- New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR.
+- PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished).
 
 If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE)
 or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR.
@@ -18,7 +19,7 @@ or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR.
 
 Best start by reading the [documentation](https://www.freqtrade.io/) to get a feel for what is possible with the bot, or head straight to the [Developer-documentation](https://www.freqtrade.io/en/latest/developer/) (WIP) which should help you getting started.
 
-## Before sending the PR:
+## Before sending the PR
 
 ### 1. Run unit tests
 
@@ -114,6 +115,6 @@ Contributors may be given commit privileges. Preference will be given to those w
 1. Access to resources for cross-platform development and testing.
 1. Time to devote to the project regularly.
 
-Being a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust Freqtrade with their Exchange API keys).
+Being a Committer does not grant write permission on `develop` or `stable` for security reasons (Users trust Freqtrade with their Exchange API keys).
 
 After being Committer for some time, a Committer may be named Core Committer and given full repository access.

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.5-slim-buster
+FROM python:3.8.6-slim-buster
 
 RUN apt-get update \
     && apt-get -y install curl build-essential libssl-dev sqlite3 \
@@ -16,13 +16,14 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Install dependencies
-COPY requirements.txt requirements-common.txt requirements-hyperopt.txt /freqtrade/
+COPY requirements.txt requirements-hyperopt.txt /freqtrade/
 RUN pip install numpy --no-cache-dir \
   && pip install -r requirements-hyperopt.txt --no-cache-dir
 
 # Install and execute
 COPY . /freqtrade/
-RUN pip install -e . --no-cache-dir
+RUN pip install -e . --no-cache-dir \
+  && mkdir /freqtrade/user_data/
 ENTRYPOINT ["freqtrade"]
 # Default to trade mode
 CMD [ "trade" ]

Dockerfile.armhf

@@ -1,7 +1,7 @@
 FROM --platform=linux/arm/v7 python:3.7.7-slim-buster
 
 RUN apt-get update \
-  && apt-get -y install curl build-essential libssl-dev libatlas3-base libgfortran5 sqlite3 \
+  && apt-get -y install curl build-essential libssl-dev libffi-dev libatlas3-base libgfortran5 sqlite3 \
   && apt-get clean \
   && pip install --upgrade pip \
   && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf
@@ -17,7 +17,7 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Install dependencies
-COPY requirements.txt requirements-common.txt /freqtrade/
+COPY requirements.txt /freqtrade/
 RUN pip install numpy --no-cache-dir \
   && pip install -r requirements.txt --no-cache-dir
 

README.md

@@ -123,13 +123,12 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor
 - `/help`: Show help message
 - `/version`: Show version
 
-
 ## Development branches
 
 The project is currently setup in two main branches:
 
-- `develop` - This branch has often new features, but might also cause breaking changes.
-- `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested.
+- `develop` - This branch has often new features, but might also contain breaking changes. We try hard to keep this branch as stable as possible.
+- `stable` - This branch contains the latest stable release. This branch is generally well tested.
 - `feat/*` - These are feature branches, which are being worked on heavily. Please don't use these unless you want to test a specific feature.
 
 ## Support
@@ -172,11 +171,11 @@ Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/
 
 **Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it.
 
-**Important:** Always create your PR against the `develop` branch, not `master`.
+**Important:** Always create your PR against the `develop` branch, not `stable`.
 
 ## Requirements
 
-### Uptodate clock
+### Up-to-date clock
 
 The clock must be accurate, syncronized to a NTP server very frequently to avoid problems with communication to the exchanges.
 

build_helpers/publish_docker.sh

@@ -2,6 +2,7 @@
 
 # Replace / with _ to create a valid tag
 TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
+TAG_PLOT=${TAG}_plot
 echo "Running for ${TAG}"
 
 # Add commit and commit_message to docker container
@@ -16,6 +17,7 @@ else
     docker pull ${IMAGE_NAME}:${TAG}
     docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} .
 fi
+docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot .
 
 if [ $? -ne 0 ]; then
     echo "failed building image"
@@ -32,6 +34,7 @@ fi
 
 # Tag image for upload
 docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG
+docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT
 if [ $? -ne 0 ]; then
     echo "failed tagging image"
     return 1

config.json.example

@@ -7,7 +7,6 @@
     "timeframe": "5m",
     "dry_run": false,
     "cancel_open_orders_on_exit": false,
-    "trailing_stop": false,
     "unfilledtimeout": {
         "buy": 10,
         "sell": 30

config_binance.json.example

@@ -7,7 +7,6 @@
     "timeframe": "5m",
     "dry_run": true,
     "cancel_open_orders_on_exit": false,
-    "trailing_stop": false,
     "unfilledtimeout": {
         "buy": 10,
         "sell": 30

config_full.json.example

@@ -116,7 +116,16 @@
     "telegram": {
         "enabled": true,
         "token": "your_telegram_token",
-        "chat_id": "your_telegram_chat_id"
+        "chat_id": "your_telegram_chat_id",
+        "notification_settings": {
+            "status": "on",
+            "warning": "on",
+            "startup": "on",
+            "buy": "on",
+            "sell": "on",
+            "buy_cancel": "on",
+            "sell_cancel": "on"
+        }
     },
     "api_server": {
         "enabled": false,

config_kraken.json.example

@@ -7,7 +7,6 @@
     "timeframe": "5m",
     "dry_run": true,
     "cancel_open_orders_on_exit": false,
-    "trailing_stop": false,
     "unfilledtimeout": {
         "buy": 10,
         "sell": 30

docker-compose.develop.yml

@@ -1,20 +0,0 @@
----
-version: '3'
-services:
-  freqtrade_develop:
-    build:
-      context: .
-      dockerfile: "./Dockerfile.develop"
-    volumes:
-      - ".:/freqtrade"
-    entrypoint:
-      - "freqtrade"
-
-  freqtrade_bash:
-    build:
-      context: .
-      dockerfile: "./Dockerfile.develop"
-    volumes:
-      - ".:/freqtrade"
-    entrypoint:
-      - "/bin/bash"

docker-compose.yml

@@ -2,8 +2,10 @@
 version: '3'
 services:
   freqtrade:
-    image: freqtradeorg/freqtrade:master
+    image: freqtradeorg/freqtrade:stable
     # image: freqtradeorg/freqtrade:develop
+    # Use plotting image
+    # image: freqtradeorg/freqtrade:develop_plot
     # Build step - only needed when additional dependencies are needed
     # build:
     #   context: .

docker/Dockerfile.develop

@@ -2,6 +2,7 @@ FROM freqtradeorg/freqtrade:develop
 
 # Install dependencies
 COPY requirements-dev.txt /freqtrade/
+
 RUN pip install numpy --no-cache-dir \
   && pip install -r requirements-dev.txt --no-cache-dir
 

docker/Dockerfile.jupyter

@@ -0,0 +1,7 @@
+FROM freqtradeorg/freqtrade:develop_plot
+
+
+RUN pip install jupyterlab --no-cache-dir
+
+# Empty the ENTRYPOINT to allow all commands
+ENTRYPOINT []

docker/Dockerfile.plot

@@ -0,0 +1,10 @@
+ARG sourceimage=develop
+FROM freqtradeorg/freqtrade:${sourceimage}
+
+# Install dependencies
+COPY requirements-plot.txt /freqtrade/
+
+RUN pip install -r requirements-plot.txt --no-cache-dir
+
+# Empty the ENTRYPOINT to allow all commands
+ENTRYPOINT []

docker/docker-compose-jupyter.yml

@@ -0,0 +1,16 @@
+---
+version: '3'
+services:
+  ft_jupyterlab:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile.jupyter
+    restart: unless-stopped
+    container_name: freqtrade
+    ports:
+      - "127.0.0.1:8888:8888"
+    volumes:
+      - "./user_data:/freqtrade/user_data"
+    # Default command used when running `docker compose up`
+    command: >
+      jupyter lab --port=8888 --ip 0.0.0.0 --allow-root

docs/backtesting.md

@@ -157,17 +157,32 @@ A backtesting result will look like that:
 | ADA/BTC  |      1 |           0.89 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |     1 |      0 |       0 |
 | LTC/BTC  |      1 |           0.68 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |     1 |      0 |       0 |
 | TOTAL    |      2 |           0.78 |           1.57 |       0.00007855 |           0.78 | 4:00:00        |     2 |      0 |       0 |
+=============== SUMMARY METRICS ===============
+| Metric                | Value               |
+|-----------------------+---------------------|
+| Backtesting from      | 2019-01-01 00:00:00 |
+| Backtesting to        | 2019-05-01 00:00:00 |
+| Total trades          | 429                 |
+| First trade           | 2019-01-01 18:30:00 |
+| First trade Pair      | EOS/USDT            |
+| Total Profit %        | 152.41%             |
+| Trades per day        | 3.575               |
+| Best day              | 25.27%              |
+| Worst day             | -30.67%             |
+| Avg. Duration Winners | 4:23:00             |
+| Avg. Duration Loser   | 6:55:00             |
+|                       |                     |
+| Max Drawdown          | 50.63%              |
+| Drawdown Start        | 2019-02-15 14:10:00 |
+| Drawdown End          | 2019-04-11 18:15:00 |
+| Market change         | -5.88%              |
+===============================================
 ```
 
+### Backtesting report table
+
 The 1st table contains all trades the bot made, including "left open trades".
 
-The 2nd table contains a recap of sell reasons.
-This table can tell you which area needs some additional work (i.e. all `sell_signal` trades are losses, so we should disable the sell-signal or work on improving that).
-
-The 3rd table contains all trades the bot had to `forcesell` at the end of the backtest period to present a full picture.
-This is necessary to simulate realistic behaviour, since the backtest period has to end at some point, while realistically, you could leave the bot running forever.
-These trades are also included in the first table, but are extracted separately for clarity.
-
 The last line will give you the overall performance of your strategy,
 here:
 
@@ -196,6 +211,58 @@ On the other hand, if you set a too high `minimal_roi` like `"0":  0.55`
 (55%), there is almost no chance that the bot will ever reach this profit.
 Hence, keep in mind that your performance is an integral mix of all different elements of the strategy, your configuration, and the crypto-currency pairs you have set up.
 
+### Sell reasons table
+
+The 2nd table contains a recap of sell reasons.
+This table can tell you which area needs some additional work (e.g. all or many of the `sell_signal` trades are losses, so you should work on improving the sell signal, or consider disabling it).
+
+### Left open trades table
+
+The 3rd table contains all trades the bot had to `forcesell` at the end of the backtesting period to present you the full picture.
+This is necessary to simulate realistic behavior, since the backtest period has to end at some point, while realistically, you could leave the bot running forever.
+These trades are also included in the first table, but are also shown separately in this table for clarity.
+
+### Summary metrics
+
+The last element of the backtest report is the summary metrics table.
+It contains some useful key metrics about performance of your strategy on backtesting data.
+
+```
+=============== SUMMARY METRICS ===============
+| Metric                | Value               |
+|-----------------------+---------------------|
+| Backtesting from      | 2019-01-01 00:00:00 |
+| Backtesting to        | 2019-05-01 00:00:00 |
+| Total trades          | 429                 |
+| First trade           | 2019-01-01 18:30:00 |
+| First trade Pair      | EOS/USDT            |
+| Total Profit %        | 152.41%             |
+| Trades per day        | 3.575               |
+| Best day              | 25.27%              |
+| Worst day             | -30.67%             |
+| Avg. Duration Winners | 4:23:00             |
+| Avg. Duration Loser   | 6:55:00             |
+|                       |                     |
+| Max Drawdown          | 50.63%              |
+| Drawdown Start        | 2019-02-15 14:10:00 |
+| Drawdown End          | 2019-04-11 18:15:00 |
+| Market change         | -5.88%              |
+===============================================
+
+```
+
+- `Total trades`: Identical to the total trades of the backtest output table.
+- `First trade`: First trade entered.
+- `First trade pair`: Which pair was part of the first trade.
+- `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option).
+- `Total Profit %`: Total profit per stake amount. Aligned to the TOTAL column of the first table.
+- `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy).
+- `Best day` / `Worst day`: Best and worst day based on daily profit.
+- `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades.
+- `Max Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced).
+- `Drawdown Start` / `Drawdown End`: Start and end datetimes for this largest drawdown (can also be visualized via the `plot-dataframe` sub-command).
+- `Market change`: Change of the market during the backtest period. Calculated as average of all pairs changes from the first to the last candle using the "close" column.
+
 ### Assumptions made by backtesting
 
 Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions:

docs/bot-usage.md

@@ -5,6 +5,9 @@ This page explains the different parameters of the bot and how to run it.
 !!! Note
     If you've used `setup.sh`, don't forget to activate your virtual environment (`source .env/bin/activate`) before running freqtrade commands.
 
+!!! Warning "Up-to-date clock"
+    The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges.
+
 ## Bot commands
 
 ```

docs/configuration.md

@@ -55,9 +55,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*  <br> **Datatype:** Boolean
 | `minimal_roi` | **Required.** Set the threshold as ratio the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Dict
 | `stoploss` |  **Required.** Value as ratio of the stoploss used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).  <br> **Datatype:** Float (as ratio)
-| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Boolean
-| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Float
-| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0` (no offset).* <br> **Datatype:** Float
+| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md#trailing-stop-loss). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Boolean
+| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-custom-positive-loss). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Float
+| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0` (no offset).* <br> **Datatype:** Float
 | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*  <br> **Datatype:** Boolean
 | `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
 | `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
@@ -278,24 +278,13 @@ This allows to buy using limit orders, sell using
 limit-orders, and create stoplosses using market orders. It also allows to set the
 stoploss "on exchange" which means stoploss order would be placed immediately once
 the buy order is fulfilled.
-If `stoploss_on_exchange` and `trailing_stop` are both set, then the bot will use `stoploss_on_exchange_interval` to check and update the stoploss on exchange periodically.
-`order_types` can be set in the configuration file or in the strategy.
+    
 `order_types` set in the configuration file overwrites values set in the strategy as a whole, so you need to configure the whole `order_types` dictionary in one place.
 
 If this is configured, the following 4 values (`buy`, `sell`, `stoploss` and
 `stoploss_on_exchange`) need to be present, otherwise the bot will fail to start.
 
-`emergencysell` is an optional value, which defaults to `market` and is used when creating stoploss on exchange orders fails.
-The below is the default which is used if this is not configured in either strategy or configuration file.
-
-Not all Exchanges support `stoploss_on_exchange`. If an exchange supports both limit and market stoploss orders, then the value of `stoploss` will be used to determine the stoploss type.
-
-If `stoploss_on_exchange` uses limit orders, the exchange needs 2 prices, the stoploss_price and the Limit price.
-`stoploss` defines the stop-price - and limit should be slightly below this.
-
-This defaults to 0.99 / 1% (configurable via `stoploss_on_exchange_limit_ratio`).
-Calculation example: we bought the asset at 100$.
-Stop-price is 95$, then limit would be `95 * 0.99 = 94.05$` - so the stoploss will happen between 95$ and 94.05$.
+For information on (`emergencysell`,`stoploss_on_exchange`,`stoploss_on_exchange_interval`,`stoploss_on_exchange_limit_ratio`) please see stop loss documentation [stop loss on exchange](stoploss.md)
 
 Syntax for Strategy:
 
@@ -386,7 +375,7 @@ Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports
 exchange markets and trading APIs. The complete up-to-date list can be found in the
 [CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python).
  However, the bot was tested by the development team with only Bittrex, Binance and Kraken,
- so the these are the only officially supported exhanges:
+ so the these are the only officially supported exchanges:
 
 - [Bittrex](https://bittrex.com/): "bittrex"
 - [Binance](https://www.binance.com/): "binance"
@@ -663,24 +652,28 @@ Filters low-value coins which would not allow setting stoplosses.
 #### PriceFilter
 
 The `PriceFilter` allows filtering of pairs by price. Currently the following price filters are supported:
+
 * `min_price`
 * `max_price`
 * `low_price_ratio`
 
 The `min_price` setting removes pairs where the price is below the specified price. This is useful if you wish to avoid trading very low-priced pairs.
-This option is disabled by default, and will only apply if set to <> 0.
+This option is disabled by default, and will only apply if set to > 0.
 
 The `max_price` setting removes pairs where the price is above the specified price. This is useful if you wish to trade only low-priced pairs.
-This option is disabled by default, and will only apply if set to <> 0.
+This option is disabled by default, and will only apply if set to > 0.
 
 The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio.
-This option is disabled by default, and will only apply if set to <> 0.
+This option is disabled by default, and will only apply if set to > 0.
+
+For `PriceFiler` at least one of its `min_price`, `max_price` or `low_price_ratio` settings must be applied.
 
 Calculation example:
 
-Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.00000012 - which is almost 10% higher than the previous value.
+Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 - one price step above would be 0.00000012, which is ~9% higher than the previous price value. You may filter out this pair by using PriceFilter with `low_price_ratio` set to 0.09 (9%) or with `min_price` set to 0.00000011, correspondingly.
 
-These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses.
+!!! Warning "Low priced pairs"
+    Low priced pairs with high "1 pip movements" are dangerous since they are often illiquid and it may also be impossible to place the desired stoploss, which can often result in high losses since price needs to be rounded to the next tradable price - so instead of having a stoploss of -5%, you could end up with a stoploss of -9% simply due to price rounding.
 
 #### ShuffleFilter
 

docs/data-analysis.md

@@ -1,12 +1,22 @@
 # Analyzing bot data with Jupyter notebooks  
 
-You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/`.  
+You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/` after initializing the user directory with `freqtrade create-userdir --userdir user_data`.  
 
-## Pro tips  
+## Quick start with docker
+
+Freqtrade provides a docker-compose file which starts up a jupyter lab server.
+You can run this server using the following command: `docker-compose -f docker/docker-compose-jupyter.yml up`
+
+This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`.
+Please use the link that's printed in the console after startup for simplified login.
+
+For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analayis-using-docker-compose) section.
+
+### Pro tips
 
 * See [jupyter.org](https://jupyter.org/documentation) for usage instructions.
 * Don't forget to start a Jupyter notebook server from within your conda or venv environment or use [nb_conda_kernels](https://github.com/Anaconda-Platform/nb_conda_kernels)*
-* Copy the example notebook before use so your changes don't get clobbered with the next freqtrade update.
+* Copy the example notebook before use so your changes don't get overwritten with the next freqtrade update.
 
 ### Using virtual environment with system-wide Jupyter installation
 
@@ -28,10 +38,8 @@ ipython kernel install --user --name=freqtrade
 !!! Note
     This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get jupyter notebooks up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community).
 
-    
-## Fine print  
-
-Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually.
+!!! Warning
+    Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually.
 
 ## Recommended workflow  
 

docs/data-download.md

@@ -8,82 +8,121 @@ If no additional parameter is specified, freqtrade will download data for `"1m"`
 Exchange and pairs will come from `config.json` (if specified using `-c/--config`).
 Otherwise `--exchange` becomes mandatory.
 
+You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101`). For incremental downloads, the relative approach should be used.
+
 !!! Tip "Tip: Updating existing data"
     If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data.
-    Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded.
+    Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded.
 
 ### Usage
 
 ```
-usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]]
-                               [--pairs-file FILE] [--days INT] [--dl-trades] [--exchange EXCHANGE]
-                               [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
-                               [--erase] [--data-format-ohlcv {json,jsongz}] [--data-format-trades {json,jsongz}]
-
-optional arguments:
-  -h, --help            show this help message and exit
-  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
-                        Show profits for only these pairs. Pairs are space-separated.
-  --pairs-file FILE     File containing a list of pairs to download.
-  --days INT            Download data for given number of days.
-  --dl-trades           Download trades instead of OHLCV data. The bot will resample trades to the desired timeframe as specified as
-                        --timeframes/-t.
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no config is provided.
-  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
-                        Specify which tickers to download. Space-separated list. Default: `1m 5m`.
-  --erase               Clean all existing data for the selected exchange/pairs/timeframes.
-  --data-format-ohlcv {json,jsongz}
-                        Storage format for downloaded candle (OHLCV) data. (default: `json`).
-  --data-format-trades {json,jsongz}
-                        Storage format for downloaded trades data. (default: `jsongz`).
-
-Common arguments:
-  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details.
-  -V, --version         show program's version number and exit
-  -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-`
-                        to read config from stdin.
-  -d PATH, --datadir PATH
-                        Path to directory with historical backtesting data.
-  --userdir PATH, --user-data-dir PATH
-                        Path to userdata directory.
-```
-
-### Data format
-
-Freqtrade currently supports 2 dataformats, `json` (plain "text" json files) and `jsongz` (a gzipped version of json files).
-By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data.
-
-This can be changed via the `--data-format-ohlcv` and `--data-format-trades` parameters respectivly.
-
-If the default dataformat has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well.
-
-!!! Note
-    You can convert between data-formats using the [convert-data](#subcommand-convert-data) and [convert-trade-data](#subcommand-convert-trade-data) methods.
-
-#### Subcommand convert data
-
-```
-usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [-d PATH] [--userdir PATH]
-                              [-p PAIRS [PAIRS ...]] --format-from
-                              {json,jsongz} --format-to {json,jsongz}
+                               [-p PAIRS [PAIRS ...]] [--pairs-file FILE]
+                               [--days INT] [--timerange TIMERANGE]
+                               [--dl-trades] [--exchange EXCHANGE]
+                               [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
                                [--erase]
-                              [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
+                               [--data-format-ohlcv {json,jsongz,hdf5}]
+                               [--data-format-trades {json,jsongz,hdf5}]
 
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Show profits for only these pairs. Pairs are space-
                         separated.
-  --format-from {json,jsongz}
+  --pairs-file FILE     File containing a list of pairs to download.
+  --days INT            Download data for given number of days.
+  --timerange TIMERANGE
+                        Specify what timerange of data to use.
+  --dl-trades           Download trades instead of OHLCV data. The bot will
+                        resample trades to the desired timeframe as specified
+                        as --timeframes/-t.
+  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
+                        config is provided.
+  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
+                        Specify which tickers to download. Space-separated
+                        list. Default: `1m 5m`.
+  --erase               Clean all existing data for the selected
+                        exchange/pairs/timeframes.
+  --data-format-ohlcv {json,jsongz,hdf5}
+                        Storage format for downloaded candle (OHLCV) data.
+                        (default: `json`).
+  --data-format-trades {json,jsongz,hdf5}
+                        Storage format for downloaded trades data. (default:
+                        `jsongz`).
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+
+```
+
+!!! Note "Startup period"
+    `download-data` is a strategy-independent command. The idea is to download a big chunk of data once, and then iteratively increase the amount of data stored.
+
+    For that reason, `download-data` does not care about the "startup-period" defined in a strategy. It's up to the user to download additional days if the backtest should start at a specific point in time (while respecting startup period).
+
+### Data format
+
+Freqtrade currently supports 3 data-formats for both OHLCV and trades data:
+
+* `json` (plain "text" json files)
+* `jsongz` (a gzip-zipped version of json files)
+* `hdf5` (a high performance datastore)
+
+By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data.
+
+This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively.
+To persist this change, you can should also add the following snippet to your configuration, so you don't have to insert the above arguments each time:
+
+``` jsonc
+    // ...
+    "dataformat_ohlcv": "hdf5",
+    "dataformat_trades": "hdf5",
+    // ...
+```
+
+If the default data-format has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well.
+
+!!! Note
+    You can convert between data-formats using the [convert-data](#sub-command-convert-data) and [convert-trade-data](#sub-command-convert-trade-data) methods.
+
+#### Sub-command convert data
+
+```
+usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+                              [-d PATH] [--userdir PATH]
+                              [-p PAIRS [PAIRS ...]] --format-from
+                              {json,jsongz,hdf5} --format-to
+                              {json,jsongz,hdf5} [--erase]
+                              [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Show profits for only these pairs. Pairs are space-
+                        separated.
+  --format-from {json,jsongz,hdf5}
                         Source format for data conversion.
-  --format-to {json,jsongz}
+  --format-to {json,jsongz,hdf5}
                         Destination format for data conversion.
   --erase               Clean all existing data for the selected
                         exchange/pairs/timeframes.
-  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
+  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
                         Specify which tickers to download. Space-separated
                         list. Default: `1m 5m`.
 
@@ -94,9 +133,10 @@ Common arguments:
                         details.
   -V, --version         show program's version number and exit
   -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`).
-                        Multiple --config options may be used. Can be set to
-                        `-` to read config from stdin.
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
   -d PATH, --datadir PATH
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
@@ -112,23 +152,23 @@ It'll also remove original json data files (`--erase` parameter).
 freqtrade convert-data --format-from json --format-to jsongz --datadir ~/.freqtrade/data/binance -t 5m 15m --erase
 ```
 
-#### Subcommand convert-trade data
+#### Sub-command convert trade data
 
 ```
 usage: freqtrade convert-trade-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                     [-d PATH] [--userdir PATH]
                                     [-p PAIRS [PAIRS ...]] --format-from
-                                    {json,jsongz} --format-to {json,jsongz}
-                                    [--erase]
+                                    {json,jsongz,hdf5} --format-to
+                                    {json,jsongz,hdf5} [--erase]
 
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Show profits for only these pairs. Pairs are space-
                         separated.
-  --format-from {json,jsongz}
+  --format-from {json,jsongz,hdf5}
                         Source format for data conversion.
-  --format-to {json,jsongz}
+  --format-to {json,jsongz,hdf5}
                         Destination format for data conversion.
   --erase               Clean all existing data for the selected
                         exchange/pairs/timeframes.
@@ -140,13 +180,15 @@ Common arguments:
                         details.
   -V, --version         show program's version number and exit
   -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`).
-                        Multiple --config options may be used. Can be set to
-                        `-` to read config from stdin.
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
   -d PATH, --datadir PATH
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
+
 ```
 
 ##### Example converting trades
@@ -158,21 +200,21 @@ It'll also remove original jsongz data files (`--erase` parameter).
 freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase
 ```
 
-### Subcommand list-data
+### Sub-command list-data
 
-You can get a list of downloaded data using the `list-data` subcommand.
+You can get a list of downloaded data using the `list-data` sub-command.
 
 ```
 usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                            [--userdir PATH] [--exchange EXCHANGE]
-                           [--data-format-ohlcv {json,jsongz}]
+                           [--data-format-ohlcv {json,jsongz,hdf5}]
                            [-p PAIRS [PAIRS ...]]
 
 optional arguments:
   -h, --help            show this help message and exit
   --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
                         config is provided.
-  --data-format-ohlcv {json,jsongz}
+  --data-format-ohlcv {json,jsongz,hdf5}
                         Storage format for downloaded candle (OHLCV) data.
                         (default: `json`).
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
@@ -194,6 +236,7 @@ Common arguments:
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
+
 ```
 
 #### Example list-data
@@ -249,15 +292,16 @@ This will download historical candle (OHLCV) data for all the currency pairs you
 ### Other Notes
 
 - To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`.
-- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust ratelimits etc.)
+- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.)
 - To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`.
 - To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days).
+- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. Eventually set end dates are ignored.
 - Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data.
 - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options.
 
 ### Trades (tick) data
 
-By default, `download-data` subcommand downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
+By default, `download-data` sub-command downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
 This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes.
 
 Since this data is large by default, the files use gzip by default. They are stored in your data-directory with the naming convention of `<pair>-trades.json.gz` (`ETH_BTC-trades.json.gz`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.

docs/deprecated.md

@@ -9,21 +9,20 @@ and are no longer supported. Please avoid their usage in your configuration.
 ### the `--refresh-pairs-cached` command line option
 
 `--refresh-pairs-cached` in the context of backtesting, hyperopt and edge allows to refresh candle data for backtesting.
-Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out 
-as a seperate freqtrade subcommand `freqtrade download-data`.
+Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out as a separate freqtrade sub-command `freqtrade download-data`.
 
-This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9 (master branch).
+This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9.
 
 ### The **--dynamic-whitelist** command line option
 
 This command line option was deprecated in 2018 and removed freqtrade 2019.6-dev (develop branch)
-and in freqtrade 2019.7 (master branch).
+and in freqtrade 2019.7.
 
 ### the `--live` command line option
 
 `--live` in the context of backtesting allowed to download the latest tick data for backtesting.
 Did only download the latest 500 candles, so was ineffective in getting good backtest data.
-Removed in 2019-7-dev (develop branch) and in freqtrade 2019-8 (master branch)
+Removed in 2019-7-dev (develop branch) and in freqtrade 2019.8.
 
 ### Allow running multiple pairlists in sequence
 
@@ -31,6 +30,6 @@ The former `"pairlist"` section in the configuration has been removed, and is re
 
 The old section of configuration parameters (`"pairlist"`) has been deprecated in 2019.11 and has been removed in 2020.4.
 
-### deprecation of bidVolume and askVolume from volumepairlist
+### deprecation of bidVolume and askVolume from volume-pairlist
 
-Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4.
+Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4, and have been removed in 2020.9.

docs/developer.md

@@ -10,13 +10,35 @@ Documentation is available at [https://freqtrade.io](https://www.freqtrade.io/)
 
 Special fields for the documentation (like Note boxes, ...) can be found [here](https://squidfunk.github.io/mkdocs-material/extensions/admonition/).
 
+To test the documentation locally use the following commands.
+
+``` bash
+pip install -r docs/requirements-docs.txt
+mkdocs serve
+```
+
+This will spin up a local server (usually on port 8000) so you can see if everything looks as you'd like it to.
+
 ## Developer setup
 
-To configure a development environment, best use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
-Alternatively (if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
+To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
+Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
 
 This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`.
 
+### Devcontainer setup
+
+The fastest and easiest way to get started is to use [VSCode](https://code.visualstudio.com/) with the Remote container extension.
+This gives developers the ability to start the bot with all required dependencies *without* needing to install any freqtrade specific dependencies on your local machine.
+
+#### Devcontainer dependencies
+
+* [VSCode](https://code.visualstudio.com/)
+* [docker](https://docs.docker.com/install/)
+* [Remote container extension documentation](https://code.visualstudio.com/docs/remote)
+
+For more information about the [Remote container extension](https://code.visualstudio.com/docs/remote), best consult the documentation.
+
 ### Tests
 
 New code should be covered by basic unittests. Depending on the complexity of the feature, Reviewers may request more in-depth unittests.
@@ -41,50 +63,37 @@ def test_method_to_test(caplog):
 
 ```
 
-### Local docker usage
+## ErrorHandling
 
-The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine.
+Freqtrade Exceptions all inherit from `FreqtradeException`.
+This general class of error should however not be used directly. Instead, multiple specialized sub-Exceptions exist.
 
-#### Install
+Below is an outline of exception inheritance hierarchy:
 
-* [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
-* [docker](https://docs.docker.com/install/)
-* [docker-compose](https://docs.docker.com/compose/install/)
-
-#### Starting the bot
-##### Use the develop dockerfile
-
-``` bash
-rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml
 ```
-
-#### Docker Compose
-
-##### Starting
-
-``` bash
-docker-compose up
++ FreqtradeException
+|
++---+ OperationalException
+|
++---+ DependencyException
+|   |
+|   +---+ PricingError
+|   |
+|   +---+ ExchangeError
+|       |
+|       +---+ TemporaryError
+|       |
+|       +---+ DDosProtection
+|       |
+|       +---+ InvalidOrderException
+|           |
+|           +---+ RetryableOrderError
+|           |
+|           +---+ InsufficientFundsError
+|
++---+ StrategyError
 ```
 
-![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png)
-
-##### Rebuilding
-
-``` bash
-docker-compose build
-```
-
-##### Execing (effectively SSH into the container)
-
-The `exec` command requires that the container already be running, if you want to start it
-that can be effected by `docker-compose up` or `docker-compose run freqtrade_develop`
-
-``` bash
-docker-compose exec freqtrade_develop /bin/bash
-```
-
-![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png)
-
 ## Modules
 
 ### Dynamic Pairlist
@@ -98,7 +107,7 @@ First of all, have a look at the [VolumePairList](https://github.com/freqtrade/f
 
 This is a simple Handler, which however serves as a good example on how to start developing.
 
-Next, modify the classname of the Handler (ideally align this with the module filename).
+Next, modify the class-name of the Handler (ideally align this with the module filename).
 
 The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`), the pairlist dedicated configuration (`self._pairlistconfig`) and the absolute position within the list of pairlists.
 
@@ -118,7 +127,7 @@ Configuration for the chain of Pairlist Handlers is done in the bot configuratio
 
 By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the pairlist. Please follow this to ensure a consistent user experience.
 
-Additional parameters can be configured as needed. For instance, `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic.
+Additional parameters can be configured as needed. For instance, `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successful and dynamic.
 
 #### short_desc
 
@@ -134,7 +143,7 @@ This is called with each iteration of the bot (only if the Pairlist Handler is a
 
 It must return the resulting pairlist (which may then be passed into the chain of Pairlist Handlers).
 
-Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected.
+Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the end-result is not shorter than expected.
 
 #### filter_pairlist
 
@@ -142,13 +151,13 @@ This method is called for each Pairlist Handler in the chain by the pairlist man
 
 This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations.
 
-It get's passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`.
+It gets passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`.
 
 The default implementation in the base class simply calls the `_validate_pair()` method for each pair in the pairlist, but you may override it. So you should either implement the `_validate_pair()` in your Pairlist Handler or override `filter_pairlist()` to do something else.
 
 If overridden, it must return the resulting pairlist (which may then be passed into the next Pairlist Handler in the chain).
 
-Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected.
+Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the end result is not shorter than expected.
 
 In `VolumePairList`, this implements different methods of sorting, does early validation so only the expected number of pairs is returned.
 
@@ -172,7 +181,7 @@ Most exchanges supported by CCXT should work out of the box.
 
 Check if the new exchange supports Stoploss on Exchange orders through their API.
 
-Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselfs. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects.
+Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselves. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects.
 
 ### Incomplete candles
 
@@ -222,13 +231,14 @@ jupyter nbconvert --ClearOutputPreprocessor.enabled=True --to markdown freqtrade
 This documents some decisions taken for the CI Pipeline.
 
 * CI runs on all OS variants, Linux (ubuntu), macOS and Windows.
-* Docker images are build for the branches `master` and `develop`.
-* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:master_pi` and `develop_pi`.
+* Docker images are build for the branches `stable` and `develop`.
+* Docker images containing Plot dependencies are also available as `stable_plot` and `develop_plot`.
+* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:stable_pi` and `develop_pi`.
 * Docker images contain a file, `/freqtrade/freqtrade_commit` containing the commit this image is based of.
 * Full docker image rebuilds are run once a week via schedule.
 * Deployments run on ubuntu.
 * ta-lib binaries are contained in the build_helpers directory to avoid fails related to external unavailability.
-* All tests must pass for a PR to be merged to `master` or `develop`.
+* All tests must pass for a PR to be merged to `stable` or `develop`.
 
 ## Creating a release
 
@@ -245,21 +255,22 @@ git checkout -b new_release <commitid>
 
 Determine if crucial bugfixes have been made between this commit and the current state, and eventually cherry-pick these.
 
+* Merge the release branch (stable) into this branch.
 * Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi.
 * Commit this part
-* push that branch to the remote and create a PR against the master branch
+* push that branch to the remote and create a PR against the stable branch
 
 ### Create changelog from git commits
 
 !!! Note
-    Make sure that the master branch is uptodate!
+    Make sure that the `stable` branch is up-to-date!
 
 ``` bash
 # Needs to be done before merging / pulling that branch.
-git log --oneline --no-decorate --no-merges master..new_release
+git log --oneline --no-decorate --no-merges stable..new_release
 ```
 
-To keep the release-log short, best wrap the full git changelog into a collapsible details secction.
+To keep the release-log short, best wrap the full git changelog into a collapsible details section.
 
 ```markdown
 <details>
@@ -272,17 +283,20 @@ To keep the release-log short, best wrap the full git changelog into a collapsib
 
 ### Create github release / tag
 
-Once the PR against master is merged (best right after merging):
+Once the PR against stable is merged (best right after merging):
 
 * Use the button "Draft a new release" in the Github UI (subsection releases).
 * Use the version-number specified as tag.
-* Use "master" as reference (this step comes after the above PR is merged).
+* Use "stable" as reference (this step comes after the above PR is merged).
 * Use the above changelog as release comment (as codeblock)
 
 ## Releases
 
 ### pypi
 
+!!! Note
+    This process is now automated as part of Github Actions.
+
 To create a pypi release, please run the following commands:
 
 Additional requirement: `wheel`, `twine` (for uploading), account on pypi with proper permissions.

docs/docker.md

@@ -1,145 +1,7 @@
-# Using Freqtrade with Docker
-
-## Install Docker
-
-Start by downloading and installing Docker CE for your platform:
-
-* [Mac](https://docs.docker.com/docker-for-mac/install/)
-* [Windows](https://docs.docker.com/docker-for-windows/install/)
-* [Linux](https://docs.docker.com/install/)
-
-Optionally, [docker-compose](https://docs.docker.com/compose/install/) should be installed and available to follow the [docker quick start guide](#docker-quick-start).
-
-Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below.
-
-## Freqtrade with docker-compose
-
-Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage.
-
-!!! Note
-    The following section assumes that docker and docker-compose is installed and available to the logged in user.
-
-!!! Note
-    All below comands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file.
-
-!!! Note "Docker on Raspberry"
-    If you're running freqtrade on a Raspberry PI, you must change the image from `freqtradeorg/freqtrade:master` to `freqtradeorg/freqtrade:master_pi` or `freqtradeorg/freqtrade:develop_pi`, otherwise the image will not work.
-
-### Docker quick start
-
-Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory.
-
-``` bash
-mkdir ft_userdata
-cd ft_userdata/
-# Download the docker-compose file from the repository
-curl https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docker-compose.yml -o docker-compose.yml
-
-# Pull the freqtrade image
-docker-compose pull
-
-# Create user directory structure
-docker-compose run --rm freqtrade create-userdir --userdir user_data
-
-# Create configuration - Requires answering interactive questions
-docker-compose run --rm freqtrade new-config --config user_data/config.json
-```
-
-The above snippet creates a new directory called "ft_userdata", downloads the latest compose file and pulls the freqtrade image.
-The last 2 steps in the snippet create the directory with user-data, as well as (interactively) the default configuration based on your selections.
-
-!!! Note
-    You can edit the configuration at any time, which is available as `user_data/config.json` (within the directory `ft_userdata`) when using the above configuration.
-
-#### Adding your strategy
-
-The configuration is now available as `user_data/config.json`.
-You should now copy your strategy to `user_data/strategies/` - and add the Strategy class name to the `docker-compose.yml` file, replacing `SampleStrategy`. If you wish to run the bot with the SampleStrategy, just leave it as it is.
-
-!!! Warning
-    The `SampleStrategy` is there for your reference and give you ideas for your own strategy.
-    Please always backtest the strategy and use dry-run for some time before risking real money!
-
-Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above).
-
-``` bash
-docker-compose up -d
-```
-
-#### Docker-compose logs
-
-Logs will be written to `user_data/logs/freqtrade.log`. 
-Alternatively, you can check the latest logs using `docker-compose logs -f`.
-
-#### Database
-
-The database will be in the user_data directory as well, and will be called `user_data/tradesv3.sqlite`.
-
-#### Updating freqtrade with docker-compose
-
-To update freqtrade when using docker-compose is as simple as running the following 2 commands:
-
-``` bash
-# Download the latest image
-docker-compose pull
-# Restart the image
-docker-compose up -d
-```
-
-This will first pull the latest image, and will then restart the container with the just pulled version.
-
-!!! Note
-    You should always check the changelog for breaking changes / manual interventions required and make sure the bot starts correctly after the update.
-
-#### Going from here
-
-Advanced users may edit the docker-compose file further to include all possible options or arguments.
-
-All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade <command> <optional arguments>`.
-
-!!! Note "`docker-compose run --rm`"
-    Including `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command).
-
-##### Example: Download data with docker-compose
-
-Download backtesting data for 5 days for the pair ETH/BTC and 1h timeframe from Binance. The data will be stored in the directory `user_data/data/` on the host.
-
-``` bash
-docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h
-```
-
-Head over to the [Data Downloading Documentation](data-download.md) for more details on downloading data.
-
-##### Example: Backtest with docker-compose
-
-Run backtesting in docker-containers for SampleStrategy and specified timerange of historical data, on 5m timeframe:
-
-``` bash
-docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m
-```
-
-Head over to the [Backtesting Documentation](backtesting.md) to learn more.
-
-#### Additional dependencies with docker-compose
-
-If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host.
-For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) for an example).
-
-You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions.
-
-``` yaml
-    image: freqtrade_custom
-    build:
-      context: .
-      dockerfile: "./Dockerfile.<yourextension>"
-```
-
-You can then run `docker-compose build` to build the docker image, and run it using the commands described above.
-
 ## Freqtrade with docker without docker-compose
 
 !!! Warning
-    The below documentation is provided for completeness and assumes that you are somewhat familiar with running docker containers. If you're just starting out with docker, we recommend to follow the [Freqtrade with docker-compose](#freqtrade-with-docker-compose) instructions.
+    The below documentation is provided for completeness and assumes that you are familiar with running docker containers. If you're just starting out with Docker, we recommend to follow the [Quickstart](docker.md) instructions.
 
 ### Download the official Freqtrade docker image
 
@@ -148,9 +10,9 @@ Pull the image from docker hub.
 Branches / tags available can be checked out on [Dockerhub tags page](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/).
 
 ```bash
-docker pull freqtradeorg/freqtrade:develop
+docker pull freqtradeorg/freqtrade:stable
 # Optionally tag the repository so the run-commands remain shorter
-docker tag freqtradeorg/freqtrade:develop freqtrade
+docker tag freqtradeorg/freqtrade:stable freqtrade
 ```
 
 To update the image, simply run the above commands again and restart your running container.
@@ -158,7 +20,7 @@ To update the image, simply run the above commands again and restart your runnin
 Should you require additional libraries, please [build the image yourself](#build-your-own-docker-image).
 
 !!! Note "Docker image update frequency"
-    The official docker images with tags `master`, `develop` and `latest` are automatically rebuild once a week to keep the base image uptodate.
+    The official docker images with tags `stable`, `develop` and `latest` are automatically rebuild once a week to keep the base image up-to-date.
     In addition to that, every merge to `develop` will trigger a rebuild for `develop` and `latest`.
 
 ### Prepare the configuration files
@@ -190,39 +52,38 @@ cp -n config.json.example config.json
 
 #### Create your database file
 
-Production
+=== "Dry-Run"
+    ``` bash
+    touch tradesv3.dryrun.sqlite
+    ```
 
-```bash
-touch tradesv3.sqlite
-````
+=== "Production"
+    ``` bash
+    touch tradesv3.sqlite
+    ```
 
-Dry-Run
 
-```bash
-touch tradesv3.dryrun.sqlite
-```
-
-!!! Note
-    Make sure to use the path to this file when starting the bot in docker.
+!!! Warning "Database File Path"
+    Make sure to use the path to the correct database file when starting the bot in Docker.
 
 ### Build your own Docker image
 
 Best start by pulling the official docker image from dockerhub as explained [here](#download-the-official-docker-image) to speed up building.
 
-To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image.
+To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image.
 
 ```bash
-docker build -t freqtrade -f Dockerfile.technical .
+docker build -t freqtrade -f docker/Dockerfile.technical .
 ```
 
-If you are developing using Docker, use `Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies:
+If you are developing using Docker, use `docker/Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies:
 
 ```bash
-docker build -f Dockerfile.develop -t freqtrade-dev .
+docker build -f docker/Dockerfile.develop -t freqtrade-dev .
 ```
 
-!!! Note
-    For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see the "5. Run a restartable docker image" section) to keep it between  updates.
+!!! Warning "Include your config file manually"
+    For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see [5. Run a restartable docker image](#run-a-restartable-docker-image)") to keep it between  updates.
 
 #### Verify the Docker image
 
@@ -243,37 +104,36 @@ docker run --rm -v `pwd`/config.json:/freqtrade/config.json -it freqtrade
 ```
 
 !!! Warning
-    In this example, the database will be created inside the docker instance and will be lost when you will refresh your image.
+    In this example, the database will be created inside the docker instance and will be lost when you refresh your image.
 
 #### Adjust timezone
 
 By default, the container will use UTC timezone.
-Should you find this irritating please add the following to your docker commands:
+If you would like to change the timezone use the following commands:
 
-##### Linux
+=== "Linux"
+    ``` bash
+    -v /etc/timezone:/etc/timezone:ro
 
-``` bash
--v /etc/timezone:/etc/timezone:ro
+    # Complete command:
+    docker run --rm -v /etc/timezone:/etc/timezone:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade
+    ```
 
-# Complete command:
-docker run --rm -v /etc/timezone:/etc/timezone:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade
-```
+=== "MacOS"
+    ```bash
+    docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade
+    ```
 
-##### MacOS
-
-There is known issue in OSX Docker versions after 17.09.1, whereby `/etc/localtime` cannot be shared causing Docker to not start. A work-around for this is to start with the following cmd.
-
-```bash
-docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade
-```
-
-More information on this docker issue and work-around can be read [here](https://github.com/docker/for-mac/issues/2396).
+!!! Note "MacOS Issues"
+    The OSX Docker versions after 17.09.1 have a known issue whereby `/etc/localtime` cannot be shared causing Docker to not start.<br>
+    A work-around for this is to start with the MacOS command above
+    More information on this docker issue and work-around can be read [here](https://github.com/docker/for-mac/issues/2396).
 
 ### Run a restartable docker image
 
 To run a restartable instance in the background (feel free to place your configuration and database files wherever it feels comfortable on your filesystem).
 
-#### Move your config file and database
+#### 1. Move your config file and database
 
 The following will assume that you place your configuration / database files to `~/.freqtrade`, which is a hidden directory in your home directory. Feel free to use a different directory and replace the directory in the upcomming commands.
 
@@ -283,7 +143,7 @@ mv config.json ~/.freqtrade
 mv tradesv3.sqlite ~/.freqtrade
 ```
 
-#### Run the docker image
+#### 2. Run the docker image
 
 ```bash
 docker run -d \

docs/docker_quickstart.md

@@ -0,0 +1,191 @@
+# Using Freqtrade with Docker
+
+## Install Docker
+
+Start by downloading and installing Docker CE for your platform:
+
+* [Mac](https://docs.docker.com/docker-for-mac/install/)
+* [Windows](https://docs.docker.com/docker-for-windows/install/)
+* [Linux](https://docs.docker.com/install/)
+
+Optionally, [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the [docker quick start guide](#docker-quick-start).
+
+Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below.
+
+## Freqtrade with docker-compose
+
+Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage.
+
+!!! Note
+    - The following section assumes that `docker` and `docker-compose` are installed and available to the logged in user.
+    - All below commands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file.
+
+### Docker quick start
+
+Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory.
+
+=== "PC/MAC/Linux"
+    ``` bash
+    mkdir ft_userdata
+    cd ft_userdata/
+    # Download the docker-compose file from the repository
+    curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml
+
+    # Pull the freqtrade image
+    docker-compose pull
+
+    # Create user directory structure
+    docker-compose run --rm freqtrade create-userdir --userdir user_data
+
+    # Create configuration - Requires answering interactive questions
+    docker-compose run --rm freqtrade new-config --config user_data/config.json
+    ```
+
+=== "RaspberryPi"
+    ``` bash
+    mkdir ft_userdata
+    cd ft_userdata/
+    # Download the docker-compose file from the repository
+    curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml
+
+    # Pull the freqtrade image
+    docker-compose pull
+
+    # Create user directory structure
+    docker-compose run --rm freqtrade create-userdir --userdir user_data
+
+    # Create configuration - Requires answering interactive questions
+    docker-compose run --rm freqtrade new-config --config user_data/config.json
+    ```
+
+    !!! Note "Change your docker Image"
+        You have to change the docker image in the docker-compose file for your Raspberry build to work properly.
+        ``` yml
+        image: freqtradeorg/freqtrade:stable_pi
+        # image: freqtradeorg/freqtrade:develop_pi
+        ```
+
+The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image.
+The last 2 steps in the snippet create the directory with `user_data`, as well as (interactively) the default configuration based on your selections.
+
+!!! Question "How to edit the bot configuration?"
+    You can edit the configuration at any time, which is available as `user_data/config.json` (within the directory `ft_userdata`) when using the above configuration.
+
+    You can also change the both Strategy and commands by editing the `docker-compose.yml` file.
+
+#### Adding a custom strategy
+
+1. The configuration is now available as `user_data/config.json`
+2. Copy a custom strategy to the directory `user_data/strategies/`
+3. add the Strategy' class name to the `docker-compose.yml` file
+
+The `SampleStrategy` is run by default.
+
+!!! Warning "`SampleStrategy` is just a demo!"
+    The `SampleStrategy` is there for your reference and give you ideas for your own strategy.
+    Please always backtest the strategy and use dry-run for some time before risking real money!
+
+Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above).
+
+``` bash
+docker-compose up -d
+```
+
+#### Docker-compose logs
+
+Logs will be located at: `user_data/logs/freqtrade.log`. 
+You can check the latest log with the command `docker-compose logs -f`.
+
+#### Database
+
+The database will be at: `user_data/tradesv3.sqlite`
+
+#### Updating freqtrade with docker-compose
+
+To update freqtrade when using `docker-compose` is as simple as running the following 2 commands:
+
+``` bash
+# Download the latest image
+docker-compose pull
+# Restart the image
+docker-compose up -d
+```
+
+This will first pull the latest image, and will then restart the container with the just pulled version.
+
+!!! Warning "Check the Changelog"
+    You should always check the changelog for breaking changes / manual interventions required and make sure the bot starts correctly after the update.
+
+### Editing the docker-compose file
+
+Advanced users may edit the docker-compose file further to include all possible options or arguments.
+
+All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade <command> <optional arguments>`.
+
+!!! Note "`docker-compose run --rm`"
+    Including `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command).
+
+#### Example: Download data with docker-compose
+
+Download backtesting data for 5 days for the pair ETH/BTC and 1h timeframe from Binance. The data will be stored in the directory `user_data/data/` on the host.
+
+``` bash
+docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h
+```
+
+Head over to the [Data Downloading Documentation](data-download.md) for more details on downloading data.
+
+#### Example: Backtest with docker-compose
+
+Run backtesting in docker-containers for SampleStrategy and specified timerange of historical data, on 5m timeframe:
+
+``` bash
+docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m
+```
+
+Head over to the [Backtesting Documentation](backtesting.md) to learn more.
+
+### Additional dependencies with docker-compose
+
+If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host.
+For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) for an example).
+
+You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions.
+
+``` yaml
+    image: freqtrade_custom
+    build:
+      context: .
+      dockerfile: "./Dockerfile.<yourextension>"
+```
+
+You can then run `docker-compose build` to build the docker image, and run it using the commands described above.
+
+## Plotting with docker-compose
+
+Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file.
+You can then use these commands as follows:
+
+``` bash
+docker-compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p BTC/ETH --timerange=20180801-20180805
+```
+
+The output will be stored in the `user_data/plot` directory, and can be opened with any modern browser.
+
+## Data analayis using docker compose
+
+Freqtrade provides a docker-compose file which starts up a jupyter lab server.
+You can run this server using the following command:
+
+``` bash
+docker-compose --rm -f docker/docker-compose-jupyter.yml up
+```
+
+This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`.
+Please use the link that's printed in the console after startup for simplified login.
+
+Since part of this image is built on your machine, it is recommended to rebuild the image from time to time to keep freqtrade (and dependencies) uptodate.
+
+``` bash
+docker-compose -f docker/docker-compose-jupyter.yml build --no-cache
+```

docs/edge.md

@@ -1,95 +1,146 @@
 # Edge positioning
 
-This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss.
+The `Edge Positioning` module uses probability to calculate your win rate and risk reward ration. It will use these statistics to control your strategy trade entry points, position side and, stoploss. 
 
 !!! Warning
-    Edge positioning is not compatible with dynamic (volume-based) whitelist.
+    `Edge positioning` is not compatible with dynamic (volume-based) whitelist.
 
 !!! Note
-    Edge does not consider anything else than buy/sell/stoploss signals. So trailing stoploss, ROI, and everything else are ignored in its calculation.
+    `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file.
+    `Edge Positioning` improves the performance of some trading strategies and *decreases* the performance of others.
 
 ## Introduction
 
-Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose.
+Trading strategies are not perfect. They are frameworks that are susceptible to the market and its indicators. Because the market is not at all predictable, sometimes a strategy will win and sometimes the same strategy will lose.
 
-But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: you give me 10$. Is it an interesting game? No, it's quite boring, isn't it?
+To obtain an edge in the market, a strategy has to make more money than it loses. Making money in trading is not only about *how often* the strategy makes or loses money. 
 
-But let's say the probability that we have heads is 80% (because our coin has the displaced distribution of mass or other defect), and the probability that we have tails is 20%. Now it is becoming interesting...
+!!! tip "It doesn't matter how often, but how much!"
+    A bad strategy might make 1 penny in *ten* transactions but lose 1 dollar in *one* transaction. If one only checks the number of winning trades, it would be misleading to think that the strategy is actually making a profit.
 
-That means 10$ X 80% versus 10$ X 20%. 8$ versus 2$. That means over time you will win 8$ risking only 2$ on each toss of coin.
+The Edge Positioning module seeks to improve a strategy's winning probability and the money that the strategy will make *on the long run*. 
 
-Let's complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% X 2$ versus 20% X 8$. It is becoming boring again because overtime you win $1.6$ (80% X 2$) and me $1.6 (20% X 8$) too.
+We raise the following question[^1]:
 
-The question is: How do you calculate that? How do you know if you wanna play?
+!!! Question "Which trade is a better option?"
+    a) A trade with 80% of chance of losing $100 and 20% chance of winning $200<br/>
+    b) A trade with 100% of chance of losing $30
 
-The answer comes to two factors:
+???+ Info "Answer"
+    The expected value of *a)* is smaller than the expected value of *b)*.<br/>
+    Hence, *b*) represents a smaller loss in the long run.<br/>
+    However, the answer is: *it depends*
 
-- Win Rate
-- Risk Reward Ratio
+Another way to look at it is to ask a similar question:
 
-### Win Rate
+!!! Question "Which trade is a better option?"
+    a) A trade with 80% of chance of winning 100 and 20% chance of losing $200<br/>
+    b) A trade with 100% of chance of winning $30
 
-Win Rate (*W*) is is the mean over some amount of trades (*N*) what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only if you won or not).
+Edge positioning tries to answer the hard questions about risk/reward and position size automatically, seeking to minimizes the chances of losing of a given strategy.
 
-```
-W = (Number of winning trades) / (Total number of trades) = (Number of winning trades) / N
-```
+### Trading, winning and losing
 
-Complementary Loss Rate (*L*) is defined as
+Let's call $o$ the return of a single transaction $o$ where $o \in \mathbb{R}$. The collection $O = \{o_1, o_2, ..., o_N\}$ is the set of all returns of transactions made during a trading session. We say that $N$ is the cardinality of $O$, or, in lay terms, it is the number of transactions made in a trading session.
 
-```
-L = (Number of losing trades) / (Total number of trades) = (Number of losing trades) / N
-```
+!!! Example
+    In a session where a strategy made three transactions we can say that $O = \{3.5, -1, 15\}$. That means that $N = 3$ and $o_1 = 3.5$, $o_2 = -1$, $o_3 = 15$.
 
-or, which is the same, as
+A winning trade is a trade where a strategy *made* money. Making money means that the strategy closed the position in a value that returned a profit, after all deducted fees. Formally, a winning trade will have a return $o_i > 0$. Similarly, a losing trade will have a return $o_j \leq 0$. With that, we can discover the set of all winning trades, $T_{win}$, as follows:
 
-```
-L = 1 – W
-```
+$$ T_{win} = \{ o \in O | o > 0 \} $$
+
+Similarly, we can discover the set of losing trades $T_{lose}$ as follows:
+
+$$ T_{lose} = \{o \in O | o \leq 0\} $$
+
+!!! Example
+    In a section where a strategy made three transactions $O = \{3.5, -1, 15, 0\}$:<br>
+    $T_{win} = \{3.5, 15\}$<br>
+    $T_{lose} = \{-1, 0\}$<br>
+
+### Win Rate and Lose Rate
+
+The win rate $W$ is the proportion of winning trades with respect to all the trades made by a strategy. We use the following function to compute the win rate:
+
+$$W = \frac{|T_{win}|}{N}$$
+
+Where $W$ is the win rate, $N$ is the number of trades and, $T_{win}$ is the set of all trades where the strategy made money.
+
+Similarly, we can compute the rate of losing trades:
+
+$$ 
+    L = \frac{|T_{lose}|}{N} 
+$$
+
+Where $L$ is the lose rate, $N$ is the amount of trades made and, $T_{lose}$ is the set of all trades where the strategy lost money. Note that the above formula is the same as calculating $L = 1 – W$ or $W = 1 – L$
 
 ### Risk Reward Ratio
 
-Risk Reward Ratio (*R*) is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose:
+Risk Reward Ratio ($R$) is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose. Formally:
 
-```
-R = Profit / Loss
-```
+$$ R = \frac{\text{potential_profit}}{\text{potential_loss}} $$
 
-Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades:
+???+ Example "Worked example of $R$ calculation"
+    Let's say that you think that the price of *stonecoin* today is $10.0. You believe that, because they will start mining stonecoin, it will go up to $15.0 tomorrow. There is the risk that the stone is too hard, and the GPUs can't mine it, so the price might go to $0 tomorrow. You are planning to invest $100.<br>
+    Your potential profit is calculated as:<br>
+    $\begin{aligned} 
+        \text{potential_profit} &= (\text{potential_price} - \text{cost_per_unit}) * \frac{\text{investment}}{\text{cost_per_unit}} \\
+                                &= (15 - 10) * \frac{100}{15}\\
+                                &= 33.33
+    \end{aligned}$<br>
+    Since the price might go to $0, the $100 dolars invested could turn into 0. We can compute the Risk Reward Ratio as follows:<br>
+    $\begin{aligned}
+        R   &= \frac{\text{potential_profit}}{\text{potential_loss}}\\
+            &= \frac{33.33}{100}\\
+            &= 0.333...
+    \end{aligned}$<br>
+    What it effectivelly means is that the strategy have the potential to make $0.33 for each $1 invested. 
 
-```
-Average profit = (Sum of profits) / (Number of winning trades)
+On a long horizon, that is, on many trades, we can calculate the risk reward by dividing the strategy' average profit on winning trades by the strategy' average loss on losing trades. We can calculate the average profit, $\mu_{win}$, as follows:
 
-Average loss = (Sum of losses) / (Number of losing trades)
+$$ \text{average_profit} = \mu_{win} = \frac{\text{sum_of_profits}}{\text{count_winning_trades}} = \frac{\sum^{o \in T_{win}} o}{|T_{win}|} $$
+
+Similarly, we can calculate the average loss, $\mu_{lose}$, as follows:
+
+$$ \text{average_loss} = \mu_{lose} = \frac{\text{sum_of_losses}}{\text{count_losing_trades}} = \frac{\sum^{o \in T_{lose}} o}{|T_{lose}|} $$
+
+Finally, we can calculate the Risk Reward ratio, $R$, as follows:
+
+$$ R = \frac{\text{average_profit}}{\text{average_loss}} = \frac{\mu_{win}}{\mu_{lose}}\\ $$
+
+
+???+ Example "Worked example of $R$ calculation using mean profit/loss"
+    Let's say the strategy that we are using makes an average win $\mu_{win} = 2.06$ and an average loss $\mu_{loss} = 4.11$.<br>
+    We calculate the risk reward ratio as follows:<br>
+    $R = \frac{\mu_{win}}{\mu_{loss}} = \frac{2.06}{4.11} = 0.5012...$
     
-R = (Average profit) / (Average loss)
-```
 
 ### Expectancy
 
-At this point we can combine *W* and *R* to create an expectancy ratio. This is a simple process of multiplying the risk reward ratio by the percentage of winning trades and subtracting the percentage of losing trades, which is calculated as follows:
+By combining the Win Rate $W$ and and the Risk Reward ratio $R$ to create an expectancy ratio $E$. A expectance ratio is the expected return of the investment made in a trade. We can compute the value of $E$ as follows:
 
-```
-Expectancy Ratio = (Risk Reward Ratio X Win Rate) – Loss Rate = (R X W) – L
-```
+$$E = R * W - L$$
 
-So lets say your Win rate is 28% and your Risk Reward Ratio is 5:
+!!! Example "Calculating $E$"
+    Let's say that a strategy has a win rate $W = 0.28$ and a risk reward ratio $R = 5$. What this means is that the strategy is expected to make 5 times the investment around on 28% of the trades it makes. Working out the example:<br>
+    $E = R * W - L = 5 * 0.28 - 0.72 = 0.68$
+    <br>
 
-```
-Expectancy = (5 X 0.28) – 0.72 = 0.68
-```
+The expectancy worked out in the example above means that, on average, this strategy' trades will return 1.68 times the size of its losses. Said another way, the strategy makes $1.68 for every $1 it loses, on average. 
 
-Superficially, this means that on average you expect this strategy’s trades to return 1.68 times the size of your loses. Said another way, you can expect to win $1.68 for every $1 you lose. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ.
+This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ.
 
 It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future.
 
 You can also use this value to evaluate the effectiveness of modifications to this system.
 
-**NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data, there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades.
+!!! Note
+    It's important to keep in mind that Edge is testing your expectancy using historical data, there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades.
 
 ## How does it work?
 
-If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over *N* trades for each stoploss. Here is an example:
+Edge combines dynamic stoploss, dynamic positions, and whitelist generation into one isolated module which is then applied to the trading strategy. If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over *N* trades for each stoploss. Here is an example:
 
 | Pair   |      Stoploss      |  Win Rate | Risk Reward Ratio | Expectancy |
 |----------|:-------------:|-------------:|------------------:|-----------:|
@@ -98,13 +149,13 @@ If enabled in config, Edge will go through historical data with a range of stopl
 | XZC/ETH  |  -0.03        |   0.52       |1.359670           | 0.228      |
 | XZC/ETH  |  -0.04        |   0.51       |1.234539           | 0.117      |
 
-The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. In the above example stoploss at 3% leads to the maximum expectancy according to historical data.
+The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. In the above example stoploss at $3%$ leads to the maximum expectancy according to historical data.
 
 Edge module then forces stoploss value it evaluated to your strategy dynamically.
 
 ### Position size
 
-Edge also dictates the stake amount for each trade to the bot according to the following factors:
+Edge dictates the amount at stake for each trade to the bot according to the following factors:
 
 - Allowed capital at risk
 - Stoploss
@@ -115,9 +166,9 @@ Allowed capital at risk is calculated as follows:
 Allowed capital at risk = (Capital available_percentage) X (Allowed risk per trade)
 ```
 
-Stoploss is calculated as described above against historical data.
+Stoploss is calculated as described above with respect to historical data.
 
-Your position size then will be:
+The position size is calculated as follows:
 
 ```
 Position size = (Allowed capital at risk) / Stoploss
@@ -125,19 +176,23 @@ Position size = (Allowed capital at risk) / Stoploss
 
 Example:
 
-Let's say the stake currency is ETH and you have 10 ETH on the exchange, your capital available percentage is 50% and you would allow 1% of risk for each trade. thus your available capital for trading is **10 x 0.5 = 5 ETH** and allowed capital at risk would be **5 x 0.01 = 0.05 ETH**.
+Let's say the stake currency is **ETH** and there is $10$ **ETH** on the wallet. The capital available percentage is $50%$ and the allowed risk per trade is $1\%$. Thus, the available capital for trading is $10 * 0.5 = 5$ **ETH** and the allowed capital at risk would be $5 * 0.01 = 0.05$ **ETH**.
 
-Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.05 / 0.02 = 2.5 ETH**.
+-   **Trade 1:** The strategy detects a new buy signal in the **XLM/ETH** market. `Edge Positioning` calculates a stoploss of $2\%$ and a position of $0.05 / 0.02 = 2.5$ **ETH**. The bot takes a position of $2.5$ **ETH** in the **XLM/ETH** market.
 
-Bot takes a position of 2.5 ETH on XLM/ETH (call it trade 1). Up next, you receive another buy signal while trade 1 is still open. This time on **BTC/ETH** market. Edge calculated stoploss for this market at 4%. So your position size would be 0.05 / 0.04 = 1.25 ETH (call it trade 2).
+-   **Trade 2:** The strategy detects a buy signal on the **BTC/ETH** market while **Trade 1** is still open. `Edge Positioning` calculates the stoploss of $4\%$ on this market. Thus, **Trade 2** position size is $0.05 / 0.04 = 1.25$ **ETH**.
 
-Note that available capital for trading didn’t change for trade 2 even if you had already trade 1. The available capital doesn’t mean the free amount on your wallet.
+!!! Tip "Available Capital $\neq$ Available in wallet"
+    The available capital for trading didn't change in **Trade 2** even with **Trade 1** still open. The available capital **is not** the free amount in the wallet.
 
-Now you have two trades open. The bot receives yet another buy signal for another market: **ADA/ETH**. This time the stoploss is calculated at 1%. So your position size is **0.05 / 0.01 = 5 ETH**. But there are already 3.75 ETH blocked in two previous trades. So the position size for this third trade would be **5 – 3.75 = 1.25 ETH**.
+-   **Trade 3:** The strategy detects a buy signal in the **ADA/ETH** market. `Edge Positioning` calculates a stoploss of $1\%$ and a position of $0.05 / 0.01 = 5$ **ETH**. Since **Trade 1** has $2.5$ **ETH** blocked and **Trade 2** has $1.25$ **ETH** blocked, there is only $5 - 1.25 - 2.5 = 1.25$ **ETH** available. Hence, the position size of **Trade 3** is $1.25$ **ETH**. 
 
-Available capital doesn’t change before a position is sold. Let’s assume that trade 1 receives a sell signal and it is sold with a profit of 1 ETH. Your total capital on exchange would be 11 ETH and the available capital for trading becomes 5.5 ETH.
+!!! Tip "Available Capital Updates"
+    The available capital does not change before a position is sold. After a trade is closed the Available Capital goes up if the trade was profitable or goes down if the trade was a loss.
 
-So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be **0.055 / 0.02 = 2.75 ETH**.
+-   The strategy detects a sell signal in the **XLM/ETH** market. The bot exits **Trade 1** for a profit of $1$ **ETH**. The total capital in the wallet becomes $11$ **ETH** and the available capital for trading becomes $5.5$ **ETH**.
+
+-   **Trade 4** The strategy detects a new buy signal int the **XLM/ETH** market. `Edge Positioning` calculates the stoploss of $2%$, and the position size of $0.055 / 0.02 = 2.75$ **ETH**.
 
 ## Configurations
 
@@ -168,23 +223,29 @@ freqtrade edge
 
 An example of its output:
 
-| pair      |   stoploss |   win rate |   risk reward ratio |   required risk reward |   expectancy |   total number of trades |   average duration (min) |
-|:----------|-----------:|-----------:|--------------------:|-----------------------:|-------------:|-------------------------:|-------------------------:|
-| AGI/BTC   |      -0.02 |       0.64 |                5.86 |                   0.56 |         3.41 |                       14 |                       54 |
-| NXS/BTC   |      -0.03 |       0.64 |                2.99 |                   0.57 |         1.54 |                       11 |                       26 |
-| LEND/BTC  |      -0.02 |       0.82 |                2.05 |                   0.22 |         1.50 |                       11 |                       36 |
-| VIA/BTC   |      -0.01 |       0.55 |                3.01 |                   0.83 |         1.19 |                       11 |                       48 |
-| MTH/BTC   |      -0.09 |       0.56 |                2.82 |                   0.80 |         1.12 |                       18 |                       52 |
-| ARDR/BTC  |      -0.04 |       0.42 |                3.14 |                   1.40 |         0.73 |                       12 |                       42 |
-| BCPT/BTC  |      -0.01 |       0.71 |                1.34 |                   0.40 |         0.67 |                       14 |                       30 |
-| WINGS/BTC |      -0.02 |       0.56 |                1.97 |                   0.80 |         0.65 |                       27 |                       42 |
-| VIBE/BTC  |      -0.02 |       0.83 |                0.91 |                   0.20 |         0.59 |                       12 |                       35 |
-| MCO/BTC   |      -0.02 |       0.79 |                0.97 |                   0.27 |         0.55 |                       14 |                       31 |
-| GNT/BTC   |      -0.02 |       0.50 |                2.06 |                   1.00 |         0.53 |                       18 |                       24 |
-| HOT/BTC   |      -0.01 |       0.17 |                7.72 |                   4.81 |         0.50 |                      209 |                        7 |
-| SNM/BTC   |      -0.03 |       0.71 |                1.06 |                   0.42 |         0.45 |                       17 |                       38 |
-| APPC/BTC  |      -0.02 |       0.44 |                2.28 |                   1.27 |         0.44 |                       25 |                       43 |
-| NEBL/BTC  |      -0.03 |       0.63 |                1.29 |                   0.58 |         0.44 |                       19 |                       59 |
+| **pair**     |   **stoploss** |   **win rate** |   **risk reward ratio** |   **required risk reward** |   **expectancy** |   **total number of trades** |   **average duration (min)** |
+|:----------|-----------:|-----------:|--------------------:|-----------------------:|-------------:|-----------------:|---------------:|
+| **AGI/BTC**   |      -0.02 |       0.64 |                5.86 |                   0.56 |         3.41 |                       14 |                       54 |
+| **NXS/BTC**   |      -0.03 |       0.64 |                2.99 |                   0.57 |         1.54 |                       11 |                       26 |
+| **LEND/BTC**  |      -0.02 |       0.82 |                2.05 |                   0.22 |         1.50 |                       11 |                       36 |
+| **VIA/BTC**   |      -0.01 |       0.55 |                3.01 |                   0.83 |         1.19 |                       11 |                       48 |
+| **MTH/BTC**   |      -0.09 |       0.56 |                2.82 |                   0.80 |         1.12 |                       18 |                       52 |
+| **ARDR/BTC**  |      -0.04 |       0.42 |                3.14 |                   1.40 |         0.73 |                       12 |                       42 |
+| **BCPT/BTC**  |      -0.01 |       0.71 |                1.34 |                   0.40 |         0.67 |                       14 |                       30 |
+| **WINGS/BTC** |      -0.02 |       0.56 |                1.97 |                   0.80 |         0.65 |                       27 |                       42 |
+| **VIBE/BTC**  |      -0.02 |       0.83 |                0.91 |                   0.20 |         0.59 |                       12 |                       35 |
+| **MCO/BTC**   |      -0.02 |       0.79 |                0.97 |                   0.27 |         0.55 |                       14 |                       31 |
+| **GNT/BTC**   |      -0.02 |       0.50 |                2.06 |                   1.00 |         0.53 |                       18 |                       24 |
+| **HOT/BTC**   |      -0.01 |       0.17 |                7.72 |                   4.81 |         0.50 |                      209 |                        7 |
+| **SNM/BTC**   |      -0.03 |       0.71 |                1.06 |                   0.42 |         0.45 |                       17 |                       38 |
+| **APPC/BTC**  |      -0.02 |       0.44 |                2.28 |                   1.27 |         0.44 |                       25 |                       43 |
+| **NEBL/BTC**  |      -0.03 |       0.63 |                1.29 |                   0.58 |         0.44 |                       19 |                       59 |
+
+Edge produced the above table by comparing `calculate_since_number_of_days` to `minimum_expectancy` to find `min_trade_number` historical information based on the config file. The timerange Edge uses for its comparisons can be further limited by using the `--timerange` switch.
+
+In live and dry-run modes, after the `process_throttle_secs` has passed, Edge will again process `calculate_since_number_of_days` against `minimum_expectancy` to find `min_trade_number`. If no `min_trade_number` is found, the bot will return "whitelist empty". Depending on the trade strategy being deployed, "whitelist empty" may be return much of the time - or *all* of the time. The use of Edge may also cause trading to occur in bursts, though this is rare.
+
+If you encounter "whitelist empty" a lot, condsider tuning `calculate_since_number_of_days`, `minimum_expectancy`  and `min_trade_number` to align to the trading frequency of your strategy.
 
 ### Update cached pairs with the latest data
 
@@ -211,3 +272,6 @@ The full timerange specification:
 * Use tickframes since 2018/01/31: `--timerange=20180131-`
 * Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301`
 * Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600`
+
+
+[^1]: Question extracted from MIT Opencourseware S096 - Mathematics with applications in Finance: https://ocw.mit.edu/courses/mathematics/18-s096-topics-in-mathematics-with-applications-in-finance-fall-2013/

docs/faq.md

@@ -1,5 +1,9 @@
 # Freqtrade FAQ
 
+## Beginner Tips & Tricks
+
+* When you work with your strategy & hyperopt file you should use a proper code editor like vscode or Pycharm. A good code editor will provide syntax highlighting as well as line numbers, making it easy to find syntax errors (most likely, pointed out by Freqtrade during startup).
+
 ## Freqtrade common issues
 
 ### The bot does not start
@@ -15,10 +19,12 @@ This could have the following reasons:
 
 ### I have waited 5 minutes, why hasn't the bot made any trades yet?!
 
-Depending on the buy strategy, the amount of whitelisted coins, the
+* Depending on the buy strategy, the amount of whitelisted coins, the
 situation of the market etc, it can take up to hours to find good entry
 position for a trade. Be patient!
 
+* Or it may because of a configuration error? Best check the logs, it's usually telling you if the bot is simply not getting buy signals (only heartbeat messages), or if there is something wrong (errors / exceptions in the log).
+
 ### I have made 12 trades already, why is my total profit negative?!
 
 I understand your disappointment but unfortunately 12 trades is just
@@ -83,7 +89,7 @@ Same fix should be done in the configuration file, if order types are defined in
 
 ### How do I search the bot logs for something?
 
-By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility subcommands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout.
+By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility sub-commands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout.
 
 * In unix shells, this normally can be done as simple as:
 ```shell
@@ -108,7 +114,7 @@ and then grep it as:
 ```shell
 $ cat /path/to/mylogfile.log | grep 'something'
 ```
-or even on the fly, as the bot works and the logfile grows:
+or even on the fly, as the bot works and the log file grows:
 ```shell
 $ tail -f /path/to/mylogfile.log | grep 'something'
 ```
@@ -129,25 +135,27 @@ to find a great result (unless if you are very lucky), so you probably
 have to run it for 10.000 or more. But it will take an eternity to
 compute.
 
-We recommend you to run it at least 10.000 epochs:
+Since hyperopt uses Bayesian search, running for too many epochs may not produce greater results.
+
+It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epochs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going.
 
 ```bash
-freqtrade hyperopt -e 10000
+freqtrade hyperopt -e 1000
 ```
 
 or if you want intermediate result to see
 
 ```bash
-for i in {1..100}; do freqtrade hyperopt -e 100; done
+for i in {1..100}; do freqtrade hyperopt -e 1000; done
 ```
 
-### Why it is so long to run hyperopt?
+### Why does it take a long time to run hyperopt?
 
-Finding a great Hyperopt results takes time.
+* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
 
-If you wonder why it takes a while to find great hyperopt results
+* If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers:
 
-This answer was written during the under the release 0.15.1, when we had:
+This answer was written during the release 0.15.1, when we had:
 
 - 8 triggers
 - 9 guards: let's say we evaluate even 10 values from each
@@ -157,7 +165,14 @@ The following calculation is still very rough and not very precise
 but it will give the idea. With only these triggers and guards there is
 already 8\*10^9\*10 evaluations. A roughly total of 80 billion evals.
 Did you run 100 000 evals? Congrats, you've done roughly 1 / 100 000 th
-of the search space.
+of the search space, assuming that the bot never tests the same parameters more than once.
+
+* The time it takes to run 1000 hyperopt epochs depends on things like: The available cpu, hard-disk, ram, timeframe, timerange, indicator settings, indicator count, amount of coins that hyperopt test strategies on and the resulting trade count - which can be 650 trades in a year or 10.0000 trades depending if the strategy aims for big profits by trading rarely or for many low profit trades. 
+
+Example: 4% profit 650 times vs 0,3% profit a trade 10.000 times in a year. If we assume you set the --timerange to 365 days. 
+
+Example: 
+`freqtrade --config config.json --strategy SampleStrategy --hyperopt SampleHyperopt -e 1000 --timerange 20190601-20200601`
 
 ## Edge module
 
@@ -165,7 +180,7 @@ of the search space.
 
 The Edge module is mostly a result of brainstorming of [@mishaker](https://github.com/mishaker) and [@creslinux](https://github.com/creslinux) freqtrade team members.
 
-You can find further info on expectancy, winrate, risk management and position size in the following sources:
+You can find further info on expectancy, win rate, risk management and position size in the following sources:
 
 - https://www.tradeciety.com/ultimate-math-guide-for-traders/
 - http://www.vantharp.com/tharp-concepts/expectancy.asp

docs/hyperopt.md

@@ -229,7 +229,7 @@ Because hyperopt tries a lot of combinations to find the best parameters it will
 We strongly recommend to use `screen` or `tmux` to prevent any connection loss.
 
 ```bash
-freqtrade hyperopt --config config.json --hyperopt <hyperoptname> -e 5000 --spaces all
+freqtrade hyperopt --config config.json --hyperopt <hyperoptname> -e 500 --spaces all
 ```
 
 Use `<hyperoptname>` as the name of the custom hyperopt used.
@@ -370,6 +370,9 @@ By default, hyperopt prints colorized results -- epochs with positive profit are
 
 You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option.
 
+!!! Note "Windows and color output"
+    Windows does not support color-output nativly, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL.
+
 ### Understand Hyperopt ROI results
 
 If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table:

docs/index.md

@@ -8,7 +8,7 @@
 <!-- Place this tag where you want the button to render. -->
 <a class="github-button" href="https://github.com/freqtrade/freqtrade/fork" data-icon="octicon-repo-forked" data-size="large" aria-label="Fork freqtrade/freqtrade on GitHub">Fork</a>
 <!-- Place this tag where you want the button to render. -->
-<a class="github-button" href="https://github.com/freqtrade/freqtrade/archive/master.zip" data-icon="octicon-cloud-download" data-size="large" aria-label="Download freqtrade/freqtrade on GitHub">Download</a>
+<a class="github-button" href="https://github.com/freqtrade/freqtrade/archive/stable.zip" data-icon="octicon-cloud-download" data-size="large" aria-label="Download freqtrade/freqtrade on GitHub">Download</a>
 <!-- Place this tag where you want the button to render. -->
 <a class="github-button" href="https://github.com/freqtrade" data-size="large" aria-label="Follow @freqtrade on GitHub">Follow @freqtrade</a>
 
@@ -37,13 +37,9 @@ Freqtrade is a crypto-currency algorithmic trading software developed in python
 
 ## Requirements
 
-### Up to date clock
-
-The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges.
-
 ### Hardware requirements
 
-To run this bot we recommend you a cloud instance with a minimum of:
+To run this bot we recommend you a linux cloud instance with a minimum of:
 
 - 2GB RAM
 - 1GB disk space

docs/installation.md

@@ -18,6 +18,9 @@ Click each one for install guide:
 
  We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot), which is optional but recommended.
 
+!!! Warning "Up-to-date clock"
+    The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges.
+
 ## Quick start
 
 Freqtrade provides the Linux/MacOS Easy Installation script to install all dependencies and help you configure the bot.
@@ -28,7 +31,7 @@ Freqtrade provides the Linux/MacOS Easy Installation script to install all depen
 The easiest way to install and run Freqtrade is to clone the bot Github repository and then run the Easy Installation script, if it's available for your platform.
 
 !!! Note "Version considerations"
-    When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `master` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable).
+    When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `stable` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable).
 
 !!! Note
     Python3.6 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository.
@@ -38,11 +41,11 @@ This can be achieved with the following commands:
 ```bash
 git clone https://github.com/freqtrade/freqtrade.git
 cd freqtrade
-git checkout master  # Optional, see (1)
+# git checkout stable  # Optional, see (1)
 ./setup.sh --install
 ```
 
-(1) This command switches the cloned repository to the use of the `master` branch. It's not needed if you wish to stay on the `develop` branch. You may later switch between branches at any time with the `git checkout master`/`git checkout develop` commands.
+(1) This command switches the cloned repository to the use of the `stable` branch. It's not needed if you wish to stay on the `develop` branch. You may later switch between branches at any time with the `git checkout stable`/`git checkout develop` commands.
 
 ## Easy Installation Script (Linux/MacOS)
 
@@ -53,7 +56,7 @@ $ ./setup.sh
 usage:
 	-i,--install    Install freqtrade from scratch
 	-u,--update     Command git pull to update.
-	-r,--reset      Hard reset your develop/master branch.
+	-r,--reset      Hard reset your develop/stable branch.
 	-c,--config     Easy config generator (Will override your existing file).
 ```
 
@@ -73,12 +76,16 @@ This option will pull the last version of your current branch and update your vi
 
 ** --reset **
 
-This option will hard reset your branch (only if you are on either `master` or `develop`) and recreate your virtualenv.
+This option will hard reset your branch (only if you are on either `stable` or `develop`) and recreate your virtualenv.
 
 ** --config **
 
 DEPRECATED - use `freqtrade new-config -c config.json` instead.
 
+### Activate your virtual environment
+
+Each time you open a new terminal, you must run `source .env/bin/activate`.
+
 ------
 
 ## Custom Installation
@@ -89,34 +96,32 @@ OS Specific steps are listed first, the [Common](#common) section below is neces
 !!! Note
     Python3.6 or higher and the corresponding pip are assumed to be available.
 
-### Linux - Ubuntu 16.04
+=== "Ubuntu 16.04"
+    #### Install necessary dependencies
 
-#### Install necessary dependencies
+    ```bash
+    sudo apt-get update
+    sudo apt-get install build-essential git
+    ```
 
-```bash
-sudo apt-get update
-sudo apt-get install build-essential git
-```
+=== "RaspberryPi/Raspbian"
+    The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/) from at least September 2019.
+    This image comes with python3.7 preinstalled, making it easy to get freqtrade up and running.
 
-### Raspberry Pi / Raspbian
+    Tested using a Raspberry Pi 3 with the Raspbian Buster lite image, all updates applied.
 
-The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/) from at least September 2019.
-This image comes with python3.7 preinstalled, making it easy to get freqtrade up and running.
+    ``` bash
+    sudo apt-get install python3-venv libatlas-base-dev
+    git clone https://github.com/freqtrade/freqtrade.git
+    cd freqtrade
 
-Tested using a Raspberry Pi 3 with the Raspbian Buster lite image, all updates applied.
+    bash setup.sh -i
+    ```
 
-``` bash
-sudo apt-get install python3-venv libatlas-base-dev
-git clone https://github.com/freqtrade/freqtrade.git
-cd freqtrade
-
-bash setup.sh -i
-```
-
-!!! Note "Installation duration"
+    !!! Note "Installation duration"
         Depending on your internet speed and the Raspberry Pi version, installation can take multiple hours to complete.
 
-!!! Note
+    !!! Note
         The above does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`.
         We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerful machine.
 
@@ -169,12 +174,7 @@ Clone the git repository:
 ```bash
 git clone https://github.com/freqtrade/freqtrade.git
 cd freqtrade
-```
-
-Optionally checkout the master branch to get the latest stable release:
-
-```bash
-git checkout master
+git checkout stable
 ```
 
 #### 4. Install python dependencies
@@ -212,72 +212,18 @@ On Linux, as an optional post-installation task, you may wish to setup the bot t
 
 ------
 
-## Using Conda
+### Anaconda
 
 Freqtrade can also be installed using Anaconda (or Miniconda).
 
+!!! Note
+    This requires the [ta-lib](#1-install-ta-lib) C-library to be installed first. See below.
+
 ``` bash
 conda env create -f environment.yml
 ```
 
-!!! Note
-    This requires the [ta-lib](#1-install-ta-lib) C-library to be installed first.
-
-## Windows
-
-We recommend that Windows users use [Docker](docker.md) as this will work much easier and smoother (also more secure).
-
-If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work.
-If that is not available on your system, feel free to try the instructions below, which led to success for some.
-
-### Install freqtrade manually
-
-!!! Note
-    Make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows.
-
-!!! Hint
-    Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Conda section](#using-conda) in this document for more information.
-
-#### Clone the git repository
-
-```bash
-git clone https://github.com/freqtrade/freqtrade.git
-```
-
-#### Install ta-lib
-
-Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
-
-As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version)
-
-```cmd
->cd \path\freqtrade-develop
->python -m venv .env
->.env\Scripts\activate.bat
-REM optionally install ta-lib from wheel
-REM >pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl
->pip install -r requirements.txt
->pip install -e .
->freqtrade
-```
-
-> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222)
-
-#### Error during installation under Windows
-
-``` bash
-error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools
-```
-
-Unfortunately, many packages requiring compilation don't provide a pre-build wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use.
-
-The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or [docker](docker.md) first.
-
----
-
-Now you have an environment ready, the next step is
-[Bot Configuration](configuration.md).
-
+-----
 ## Troubleshooting 
 
 ### MacOS installation error
@@ -291,4 +237,9 @@ For MacOS 10.14, this can be accomplished with the below command.
 open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg
 ```
 
-If this file is inexistant, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details.
+If this file is inexistent, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details.
+
+-----
+
+Now you have an environment ready, the next step is
+[Bot Configuration](configuration.md).

docs/javascripts/config.js

@@ -0,0 +1,12 @@
+window.MathJax = {
+  tex: {
+    inlineMath: [["\\(", "\\)"]],
+    displayMath: [["\\[", "\\]"]],
+    processEscapes: true,
+    processEnvironments: true
+  },
+  options: {
+    ignoreHtmlClass: ".*|",
+    processHtmlClass: "arithmatex"
+  }
+};

docs/plotting.md

@@ -224,7 +224,8 @@ Possible options for the `freqtrade plot-profit` subcommand:
 
 ```
 usage: freqtrade plot-profit [-h] [-v] [--logfile FILE] [-V] [-c PATH]
-                             [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]]
+                             [-d PATH] [--userdir PATH] [-s NAME]
+                             [--strategy-path PATH] [-p PAIRS [PAIRS ...]]
                              [--timerange TIMERANGE] [--export EXPORT]
                              [--export-filename PATH] [--db-url PATH]
                              [--trade-source {DB,file}] [-i TIMEFRAME]
@@ -270,6 +271,11 @@ Common arguments:
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 
+Strategy arguments:
+  -s NAME, --strategy NAME
+                        Specify strategy class name which will be used by the
+                        bot.
+  --strategy-path PATH  Specify additional strategy lookup path.
 ```
 
 The `-p/--pairs`  argument, can be used to limit the pairs that are considered for this calculation.
@@ -279,7 +285,7 @@ Examples:
 Use custom backtest-export file
 
 ``` bash
-freqtrade plot-profit  -p LTC/BTC --export-filename user_data/backtest_results/backtest-result-Strategy005.json
+freqtrade plot-profit  -p LTC/BTC --export-filename user_data/backtest_results/backtest-result.json
 ```
 
 Use custom database

docs/requirements-docs.txt

@@ -1,2 +1,2 @@
-mkdocs-material==5.5.0
+mkdocs-material==5.5.13
 mdx_truly_sane_lists==1.2

docs/rest-api.md

@@ -46,7 +46,7 @@ secrets.token_hex()
 
 ### Configuration with docker
 
-If you run your bot using docker, you'll need to have the bot listen to incomming connections. The security is then handled by docker.
+If you run your bot using docker, you'll need to have the bot listen to incoming connections. The security is then handled by docker.
 
 ``` json
     "api_server": {
@@ -106,26 +106,30 @@ python3 scripts/rest_client.py --config rest_config.json <command> [optional par
 
 ## Available commands
 
-|  Command | Default | Description |
-|----------|---------|-------------|
-| `start` | | Starts the trader
-| `stop` | | Stops the trader
-| `stopbuy` | | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
-| `reload_config` | | Reloads the configuration file
-| `show_config` | | Shows part of the current configuration with relevant settings to operation
-| `status` | | Lists all open trades
-| `count` | | Displays number of trades used and available
-| `profit` | | Display a summary of your profit/loss from close trades and some stats about your performance
-| `forcesell <trade_id>` | | Instantly sells the given trade  (Ignoring `minimum_roi`).
-| `forcesell all` | | Instantly sells all open trades (Ignoring `minimum_roi`).
-| `forcebuy <pair> [rate]` | | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True)
-| `performance` | | Show performance of each finished trade grouped by pair
-| `balance` | | Show account balance per currency
-| `daily <n>` | 7 | Shows profit or loss per day, over the last n days
-| `whitelist` | | Show the current whitelist
-| `blacklist [pair]` | | Show the current blacklist, or adds a pair to the blacklist.
-| `edge` | | Show validated pairs by Edge if it is enabled.
-| `version` | | Show version
+|  Command | Description |
+|----------|-------------|
+| `ping` | Simple command testing the API Readiness - requires no authentication.
+| `start` | Starts the trader
+| `stop` | Stops the trader
+| `stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
+| `reload_config` | Reloads the configuration file
+| `trades` | List last trades.
+| `delete_trade <trade_id>` | Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange.
+| `show_config` | Shows part of the current configuration with relevant settings to operation
+| `logs` | Shows last log messages
+| `status` | Lists all open trades
+| `count` | Displays number of trades used and available
+| `profit` | Display a summary of your profit/loss from close trades and some stats about your performance
+| `forcesell <trade_id>` | Instantly sells the given trade  (Ignoring `minimum_roi`).
+| `forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`).
+| `forcebuy <pair> [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True)
+| `performance` | Show performance of each finished trade grouped by pair
+| `balance` | Show account balance per currency
+| `daily <n>` | Shows profit or loss per day, over the last n days (n defaults to 7)
+| `whitelist` | Show the current whitelist
+| `blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist.
+| `edge` | Show validated pairs by Edge if it is enabled.
+| `version` | Show version
 
 Possible commands can be listed from the rest-client script using the `help` command.
 
@@ -135,78 +139,83 @@ python3 scripts/rest_client.py help
 
 ``` output
 Possible commands:
+
 balance
-        Get the account balance
-        :returns: json object
+	Get the account balance.
 
 blacklist
-        Show the current blacklist
+	Show the current blacklist.
+
         :param add: List of coins to add (example: "BNB/BTC")
-        :returns: json object
 
 count
-        Returns the amount of open trades
-        :returns: json object
+	Return the amount of open trades.
 
 daily
-        Returns the amount of open trades
-        :returns: json object
+	Return the amount of open trades.
+
+delete_trade
+	Delete trade from the database.
+        Tries to close open orders. Requires manual handling of this asset on the exchange.
+
+        :param trade_id: Deletes the trade with this ID from the database.
 
 edge
-        Returns information about edge
-        :returns: json object
+	Return information about edge.
 
 forcebuy
-        Buy an asset
+	Buy an asset.
+
         :param pair: Pair to buy (ETH/BTC)
         :param price: Optional - price to buy
-        :returns: json object of the trade
 
 forcesell
-        Force-sell a trade
+	Force-sell a trade.
+
         :param tradeid: Id of the trade (can be received via status command)
-        :returns: json object
+
+logs
+	Show latest logs.
+
+        :param limit: Limits log messages to the last <limit> logs. No limit to get all the trades.
 
 performance
-        Returns the performance of the different coins
-        :returns: json object
+	Return the performance of the different coins.
 
 profit
-        Returns the profit summary
-        :returns: json object
+	Return the profit summary.
 
 reload_config
-        Reload configuration
-        :returns: json object
+	Reload configuration.
 
 show_config
+	
         Returns part of the configuration, relevant for trading operations.
-        :return: json object containing the version
 
 start
-        Start the bot if it's in stopped state.
-        :returns: json object
+	Start the bot if it's in the stopped state.
 
 status
-        Get the status of open trades
-        :returns: json object
+	Get the status of open trades.
 
 stop
-        Stop the bot. Use start to restart
-        :returns: json object
+	Stop the bot. Use `start` to restart.
 
 stopbuy
-        Stop buying (but handle sells gracefully).
-        use reload_config to reset
-        :returns: json object
+	Stop buying (but handle sells gracefully). Use `reload_config` to reset.
+
+trades
+	Return trades history.
+
+        :param limit: Limits trades to the X last trades. No limit to get all the trades.
 
 version
-        Returns the version of the bot
-        :returns: json object containing the version
+	Return the version of the bot.
 
 whitelist
-        Show the current whitelist
-        :returns: json object
+	Show the current whitelist.
+
+
 ```
 
 ## Advanced API usage using JWT tokens

docs/sandbox-testing.md

@@ -1,104 +1,59 @@
 # Sandbox API testing
 
-Where an exchange provides a sandbox for risk-free integration, or end-to-end, testing CCXT provides access to these.
+Some exchanges provide sandboxes or testbeds for risk-free testing, while running the bot against a real exchange.
+With some configuration, freqtrade (in combination with ccxt) provides access to these.
 
-This document is a *light overview of configuring Freqtrade and GDAX sandbox.
-This can be useful to developers and trader alike as Freqtrade is quite customisable.
+This document is an overview to configure Freqtrade to be used with sandboxes.
+This can be useful to developers and trader alike.
 
-When testing your API connectivity, make sure to use the following URLs.
-***Website**
-https://public.sandbox.gdax.com
-***REST API**
-https://api-public.sandbox.gdax.com
+## Exchanges known to have a sandbox / testnet
+
+* [binance](https://testnet.binance.vision/)
+* [coinbasepro](https://public.sandbox.pro.coinbase.com)
+* [gemini](https://exchange.sandbox.gemini.com/)
+* [huobipro](https://www.testnet.huobi.pro/)
+* [kucoin](https://sandbox.kucoin.com/)
+* [phemex](https://testnet.phemex.com/) 
+
+!!! Note
+    We did not test correct functioning of all of the above testnets. Please report your experiences with each sandbox.
 
 ---
 
-# Configure a Sandbox account on Gdax
+## Configure a Sandbox account
 
-Aim of this document section
+When testing your API connectivity, make sure to use the appropriate sandbox / testnet URL.
 
-- An sanbox account
-- create 2FA (needed to create an API)
-- Add test 50BTC to account 
-- Create :
-- - API-KEY
-- - API-Secret
-- - API Password
+In general, you should follow these steps to enable an exchange's sandbox:
 
-## Acccount
+* Figure out if an exchange has a sandbox (most likely by using google or the exchange's support documents)
+* Create a sandbox account (often the sandbox-account requires separate registration)
+* [Add some test assets to account](#add-test-funds)
+* Create API keys
 
-This link will redirect to the sandbox main page to login / create account dialogues:
-https://public.sandbox.pro.coinbase.com/orders/
+### Add test funds
 
-After registration and Email confimation you wil be redirected into your sanbox account.  It is easy to verify you're in sandbox by checking the URL bar.
-> https://public.sandbox.pro.coinbase.com/
+Usually, sandbox exchanges allow depositing funds directly via web-interface.
+You should make sure to have a realistic amount of funds available to your test-account, so results are representable of your real account funds.
 
-## Enable 2Fa (a prerequisite to creating sandbox API Keys)
+!!! Warning
+    Test exchanges will **NEVER** require your real credit card or banking details!
 
-From within sand box site select your profile, top right.
->Or as a direct link: https://public.sandbox.pro.coinbase.com/profile
+## Configure freqtrade to use a exchange's sandbox
 
-From the menu panel to the left of the screen select
-
-> Security: "*View or Update*"
-
-In the new site select "enable authenticator" as typical google Authenticator.
-
-- open Google Authenticator on your phone
-- scan barcode
-- enter your generated 2fa
-
-## Enable API Access
-
-From within sandbox select profile>api>create api-keys
->or as a direct link: https://public.sandbox.pro.coinbase.com/profile/api
-
-Click on "create one" and ensure  **view** and **trade**  are "checked" and sumbit your 2FA
-
-- **Copy and paste the Passphase** into a notepade this will be needed later
-- **Copy and paste the API Secret** popup into a notepad this will needed later
-- **Copy and paste the API Key** into a notepad this will needed later
-
-## Add 50 BTC test funds
-
-To add funds, use the web interface deposit and withdraw buttons.
-
-To begin select 'Wallets' from the top menu.
-> Or as a direct link: https://public.sandbox.pro.coinbase.com/wallets
-
-- Deposits (bottom left of screen)
-- - Deposit Funds Bitcoin
-- - - Coinbase BTC Wallet
-- - - - Max (50 BTC)
-- - - - - Deposit
-
-*This process may be repeated for other currencies, ETH as example*
-
----
-
-# Configure Freqtrade to use Gax Sandbox 
-
-The aim of this document section
-
-- Enable sandbox URLs in Freqtrade
-- Configure API
-- - secret
-- - key
-- - passphrase
- 
-## Sandbox URLs
+### Sandbox URLs
 
 Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade.
 These include `['test']` and `['api']`.
 
-- `[Test]` if available will point to an Exchanges sandbox. 
-- `[Api]` normally used, and resolves to live API target on the exchange 
+* `[Test]` if available will point to an Exchanges sandbox.
+* `[Api]` normally used, and resolves to live API target on the exchange.
 
 To make use of sandbox / test add "sandbox": true, to your config.json
 
 ```json
   "exchange": {
-        "name": "gdax",
+        "name": "coinbasepro",
         "sandbox": true,
         "key": "5wowfxemogxeowo;heiohgmd",
         "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==",
@@ -106,36 +61,57 @@ To make use of sandbox / test add "sandbox": true, to your config.json
         "outdated_offset": 5
         "pair_whitelist": [
             "BTC/USD"
+        ]
+  },
+  "datadir": "user_data/data/coinbasepro_sandbox"
 ```
 
-Also insert your 
+Also the following information:
 
-- api-key  (noted earlier)
-- api-secret (noted earlier)
-- password (the passphrase - noted earlier)
+* api-key (created for the sandbox webpage)
+* api-secret (noted earlier)
+* password (the passphrase - noted earlier)
+
+!!! Tip "Different data directory"
+    We also recommend to set `datadir` to something identifying downloaded data as sandbox data, to avoid having sandbox data mixed with data from the real exchange.
+    This can be done by adding the `"datadir"` key to the configuration.
+    Now, whenever you use this configuration, your data directory will be set to this directory.
 
 ---
 
 ## You should now be ready to test your sandbox
 
-Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox.
-** Typically the  BTC/USD has the most activity in sandbox to test against.
+Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox. Also make sure to select a pair which shows at least some decent value (which very often is BTC/<somestablecoin>).
 
-## GDAX - Old Candles problem
+## Common problems with sandbox exchanges
 
-It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks.
+Sandbox exchange instances often have very low volume, which can cause some problems which usually are not seen on a real exchange instance.
 
-To disable this check, add / change the `"outdated_offset"` parameter in the exchange section of your configuration to adjust for this delay.
-Example based on the above configuration:
+### Old Candles problem
 
-```json
-  "exchange": {
-        "name": "gdax",
-        "sandbox": true,
-        "key": "5wowfxemogxeowo;heiohgmd",
-        "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==",
-        "password": "1bkjfkhfhfu6sr",
-        "outdated_offset": 30
-        "pair_whitelist": [
-            "BTC/USD"
-```
+Since Sandboxes often have low volume, candles can be quite old and show no volume.
+To disable the error "Outdated history for pair ...", best increase the parameter `"outdated_offset"` to a number that seems realistic for the sandbox you're using.
+
+### Unfilled orders
+
+Sandboxes often have very low volumes - which means that many trades can go unfilled, or can go unfilled for a very long time.
+
+To mitigate this, you can try to match the first order on the opposite orderbook side using the following configuration:
+
+``` jsonc
+  "order_types": {
+    "buy": "limit",
+    "sell": "limit"
+    // ...
+  },
+  "bid_strategy": {
+    "price_side": "ask",
+    // ...
+  },
+  "ask_strategy":{
+    "price_side": "bid",
+    // ...
+  },
+  ```
+
+  The configuration is similar to the suggested configuration for market orders - however by using limit-orders you can avoid moving the price too much, and you can set the worst price you might get.

docs/sql_cheatsheet.md

@@ -46,7 +46,7 @@ sqlite3
 ### Trade table structure
 
 ```sql
-CREATE TABLE trades
+CREATE TABLE trades(
     id INTEGER NOT NULL,
     exchange VARCHAR NOT NULL,
     pair VARCHAR NOT NULL,
@@ -110,7 +110,7 @@ SET is_open=0,
   close_date=<close_date>,
   close_rate=<close_rate>,
   close_profit = close_rate / open_rate - 1,
-  close_profit_abs = (amount * <close_rate> * (1 - fee_close) - (amount * (open_rate * 1 - fee_open))),
+  close_profit_abs = (amount * <close_rate> * (1 - fee_close) - (amount * (open_rate * (1 - fee_open)))),
   sell_reason=<sell_reason>
 WHERE id=<trade_ID_to_update>;
 ```
@@ -123,7 +123,7 @@ SET is_open=0,
   close_date='2020-06-20 03:08:45.103418',
   close_rate=0.19638016,
   close_profit=0.0496,
-  close_profit_abs = (amount * 0.19638016 * (1 - fee_close) - (amount * open_rate * (1 - fee_open))),
+  close_profit_abs = (amount * 0.19638016 * (1 - fee_close) - (amount * (open_rate * (1 - fee_open)))),
   sell_reason='force_sell'  
 WHERE id=31;
 ```

docs/stoploss.md

@@ -6,7 +6,63 @@ For example, value `-0.10` will cause immediate sell if the profit dips below -1
 Most of the strategy files already include the optimal `stoploss` value.
 
 !!! Info
-    All stoploss properties mentioned in this file can be set in the Strategy, or in the configuration. Configuration values will override the strategy values.
+    All stoploss properties mentioned in this file can be set in the Strategy, or in the configuration.  
+    <ins>Configuration values will override the strategy values.</ins>
+
+## Stop Loss On-Exchange/Freqtrade
+
+Those stoploss modes can be *on exchange* or *off exchange*.
+
+These modes can be configured with these values:
+
+``` python
+    'emergencysell': 'market',
+    'stoploss_on_exchange': False
+    'stoploss_on_exchange_interval': 60,
+    'stoploss_on_exchange_limit_ratio': 0.99
+```
+
+!!! Note
+    Stoploss on exchange is only supported for Binance (stop-loss-limit), Kraken (stop-loss-market) and FTX (stop limit and stop-market) as of now.  
+    <ins>Do not set too low stoploss value if using stop loss on exchange!</ins>  
+    If set to low/tight then you have greater risk of missing fill on the order and stoploss will not work
+
+### stoploss_on_exchange and stoploss_on_exchange_limit_ratio
+Enable or Disable stop loss on exchange.
+If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled.
+
+If `stoploss_on_exchange` uses limit orders, the exchange needs 2 prices, the stoploss_price and the Limit price.  
+`stoploss` defines the stop-price where the limit order is placed - and limit should be slightly below this.  
+If an exchange supports both limit and market stoploss orders, then the value of `stoploss` will be used to determine the stoploss type.  
+
+Calculation example: we bought the asset at 100$.  
+Stop-price is 95$, then limit would be `95 * 0.99 = 94.05$` - so the limit order fill can happen between 95$ and 94.05$.  
+
+For example, assuming the stoploss is on exchange, and trailing stoploss is enabled, and the market is going up, then the bot automatically cancels the previous stoploss order and puts a new one with a stop value higher than the previous stoploss order.
+
+### stoploss_on_exchange_interval
+In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary.  
+The bot cannot do these every 5 seconds (at each iteration), otherwise it would get banned by the exchange.
+So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute).
+This same logic will reapply a stoploss order on the exchange should you cancel it accidentally.
+
+### emergencysell
+`emergencysell` is an optional value, which defaults to `market` and is used when creating stop loss on exchange orders fails.
+The below is the default which is used if not changed in strategy or configuration file.
+
+Example from strategy file:
+
+``` python
+order_types = {
+    'buy': 'limit',
+    'sell': 'limit',
+    'emergencysell': 'market',
+    'stoploss': 'market',
+    'stoploss_on_exchange': True,
+    'stoploss_on_exchange_interval': 60,
+    'stoploss_on_exchange_limit_ratio': 0.99
+}
+```
 
 ## Stop Loss Types
 
@@ -17,29 +73,29 @@ At this stage the bot contains the following stoploss support modes:
 3. Trailing stop loss, custom positive loss.
 4. Trailing stop loss only once the trade has reached a certain offset.
 
-Those stoploss modes can be *on exchange* or *off exchange*. If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled.
-
-In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary.
-
-For example, assuming the stoploss is on exchange, and trailing stoploss is enabled, and the market is going up, then the bot automatically cancels the previous stoploss order and puts a new one with a stop value higher than the previous stoploss order.
-The bot cannot do this every 5 seconds (at each iteration), otherwise it would get banned by the exchange.
-So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute).
-This same logic will reapply a stoploss order on the exchange should you cancel it accidentally.
-
-!!! Note
-    Stoploss on exchange is only supported for Binance (stop-loss-limit), Kraken (stop-loss-market) and FTX (stop limit and stop-market) as of now.
-
-## Static Stop Loss
+### Static Stop Loss
 
 This is very simple, you define a stop loss of x (as a ratio of price, i.e. x * 100% of price). This will try to sell the asset once the loss exceeds the defined loss.
 
-## Trailing Stop Loss
+Example of stop loss:
+
+``` python
+    stoploss = -0.10
+```
+
+For example, simplified math:
+* the bot buys an asset at a price of 100$
+* the stop loss is defined at -10%
+* the stop loss would get triggered once the asset drops below 90$
+
+### Trailing Stop Loss
 
 The initial value for this is `stoploss`, just as you would define your static Stop loss.
 To enable trailing stoploss:
 
 ``` python
-trailing_stop = True
+    stoploss = -0.10
+    trailing_stop = True
 ```
 
 This will now activate an algorithm, which automatically moves the stop loss up every time the price of your asset increases.
@@ -47,35 +103,43 @@ This will now activate an algorithm, which automatically moves the stop loss up
 For example, simplified math:
 
 * the bot buys an asset at a price of 100$
-* the stop loss is defined at 2%
-* the stop loss would get triggered once the asset dropps below 98$
+* the stop loss is defined at -10%
+* the stop loss would get triggered once the asset drops below 90$
 * assuming the asset now increases to 102$
-* the stop loss will now be 2% of 102$ or 99.96$
-* now the asset drops in value to 101$, the stop loss will still be 99.96$ and would trigger at 99.96$.
+* the stop loss will now be -10% of 102$ = 91.8$
+* now the asset drops in value to 101$, the stop loss will still be 91.8$ and would trigger at 91.8$.
 
-In summary: The stoploss will be adjusted to be always be 2% of the highest observed price.
+In summary: The stoploss will be adjusted to be always be -10% of the highest observed price.
 
-### Custom positive stoploss
+### Trailing stop loss, custom positive loss
 
-It is also possible to have a default stop loss, when you are in the red with your buy, but once your profit surpasses a certain percentage, the system will utilize a new stop loss, which can have a different value.
-For example your default stop loss is 5%, but once you have 1.1% profit, it will be changed to be only a 1% stop loss, which trails the green candles until it goes below them.
+It is also possible to have a default stop loss, when you are in the red with your buy (buy - fee), but once you hit positive result the system will utilize a new stop loss, which can have a different value.
+For example, your default stop loss is -10%, but once you have more than 0% profit (example 0.1%) a different trailing stoploss will be used.
 
-Both values require `trailing_stop` to be set to true.
+!!! Note
+    If you want the stoploss to only be changed when you break even of making a profit (what most users want) please refer to next section with [offset enabled](#Trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset).
+
+Both values require `trailing_stop` to be set to true and `trailing_stop_positive` with a value.
 
 ``` python
-    trailing_stop_positive = 0.01
-    trailing_stop_positive_offset = 0.011
+    stoploss = -0.10
+    trailing_stop = True
+    trailing_stop_positive = 0.02
 ```
 
-The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit.
+For example, simplified math:
+
+* the bot buys an asset at a price of 100$
+* the stop loss is defined at -10%
+* the stop loss would get triggered once the asset drops below 90$
+* assuming the asset now increases to 102$
+* the stop loss will now be -2% of 102$ = 99.96$ (99.96$ stop loss will be locked in and will follow asset price increasements with -2%)
+* now the asset drops in value to 101$, the stop loss will still be 99.96$ and would trigger at 99.96$
+
+The 0.02 would translate to a -2% stop loss.
 Before this, `stoploss` is used for the trailing stoploss.
 
-Read the [next section](#trailing-only-once-offset-is-reached) to keep stoploss at 5% of the entry point.
-
-!!! Tip
-    Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade.
-
-### Trailing only once offset is reached
+### Trailing stop loss only once the trade has reached a certain offset
 
 It is also possible to use a static stoploss until the offset is reached, and then trail the trade to take profits once the market turns.
 
@@ -87,17 +151,28 @@ This option can be used with or without `trailing_stop_positive`, but uses `trai
     trailing_only_offset_is_reached = True
 ```
 
-Simplified example:
+Configuration (offset is buyprice + 3%):
 
 ``` python
-    stoploss = 0.05
+    stoploss = -0.10
+    trailing_stop = True
+    trailing_stop_positive = 0.02
     trailing_stop_positive_offset = 0.03
     trailing_only_offset_is_reached = True
 ```
 
+For example, simplified math:
+
 * the bot buys an asset at a price of 100$
-* the stop loss is defined at 5%
-* the stop loss will remain at 95% until profit reaches +3%
+* the stop loss is defined at -10%
+* the stop loss would get triggered once the asset drops below 90$
+* stoploss will remain at 90$ unless asset increases to or above our configured offset
+* assuming the asset now increases to 103$ (where we have the offset configured)
+* the stop loss will now be -2% of 103$ = 100.94$
+* now the asset drops in value to 101$, the stop loss will still be 100.94$ and would trigger at 100.94$
+
+!!! Tip
+    Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade.
 
 ## Changing stoploss on open trades
 

docs/strategy-advanced.md

@@ -199,3 +199,24 @@ class Awesomestrategy(IStrategy):
         return True
 
 ```
+
+## Derived strategies
+
+The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched:
+
+``` python
+class MyAwesomeStrategy(IStrategy):
+    ...
+    stoploss = 0.13
+    trailing_stop = False
+    # All other attributes and methods are here as they
+    # should be in any custom strategy...
+    ...
+
+class MyAwesomeStrategy2(MyAwesomeStrategy):
+    # Override something
+    stoploss = 0.08
+    trailing_stop = True
+```
+
+Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need.

docs/strategy-customization.md

@@ -58,12 +58,12 @@ file as reference.**
 
 !!! Note "Strategies and Backtesting"
     To avoid problems and unexpected differences between Backtesting and dry/live modes, please be aware
-    that during backtesting the full time-interval is passed to the `populate_*()` methods at once.
+    that during backtesting the full time range is passed to the `populate_*()` methods at once.
     It is therefore best to use vectorized operations (across the whole dataframe, not loops) and
     avoid index referencing (`df.iloc[-1]`), but instead use `df.shift()` to get to the previous candle.
 
 !!! Warning "Warning: Using future data"
-    Since backtesting passes the full time interval to the `populate_*()` methods, the strategy author
+    Since backtesting passes the full time range to the `populate_*()` methods, the strategy author
     needs to take care to avoid having the strategy utilize data from the future.
     Some common patterns for this are listed in the [Common Mistakes](#common-mistakes-when-developing-strategies) section of this document.
 
@@ -251,7 +251,7 @@ minimal_roi = {
 While technically not completely disabled, this would sell once the trade reaches 10000% Profit.
 
 To use times based on candle duration (timeframe), the following snippet can be handy.
-This will allow you to change the ticket_interval for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...)
+This will allow you to change the timeframe for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...)
 
 ``` python
 from freqtrade.exchange import timeframe_to_minutes
@@ -285,7 +285,7 @@ If your exchange supports it, it's recommended to also set `"stoploss_on_exchang
 
 For more information on order_types please look [here](configuration.md#understand-order_types).
 
-### Timeframe (ticker interval)
+### Timeframe (formerly ticker interval)
 
 This is the set of candles the bot should download and use for the analysis.
 Common values are `"1m"`, `"5m"`, `"15m"`, `"1h"`, however all values supported by your exchange should work.
@@ -328,15 +328,15 @@ class Awesomestrategy(IStrategy):
 
 ***
 
-### Additional data (informative_pairs)
+## Additional data (informative_pairs)
 
-#### Get data for non-tradeable pairs
+### Get data for non-tradeable pairs
 
 Data for additional, informative pairs (reference pairs) can be beneficial for some strategies.
-Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below).
+OHLCV data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below).
 These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting.
 
-The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument.
+The pairs need to be specified as tuples in the format `("pair", "timeframe")`, with pair as the first and timeframe as the second argument.
 
 Sample:
 
@@ -347,15 +347,17 @@ def informative_pairs(self):
             ]
 ```
 
+A full sample can be found [in the DataProvider section](#complete-data-provider-sample).
+
 !!! Warning
     As these pairs will be refreshed as part of the regular whitelist refresh, it's best to keep this list short.
-    All intervals and all pairs can be specified as long as they are available (and active) on the used exchange.
-    It is however better to use resampling to longer time-intervals when possible
+    All timeframes and all pairs can be specified as long as they are available (and active) on the used exchange.
+    It is however better to use resampling to longer timeframes whenever possible
     to avoid hammering the exchange with too many requests and risk being blocked.
 
 ***
 
-### Additional data (DataProvider)
+## Additional data (DataProvider)
 
 The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy.
 
@@ -363,10 +365,14 @@ All methods return `None` in case of failure (do not raise an exception).
 
 Please always check the mode of operation to select the correct method to get data (samples see below).
 
-#### Possible options for DataProvider
+!!! Warning "Hyperopt"
+    Dataprovider is available during hyperopt, however it can only be used in `populate_indicators()` within a strategy.
+    It is not available in `populate_buy()` and `populate_sell()` methods, nor in `populate_indicators()`, if this method located in the hyperopt file.
 
-- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval).
-- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist)
+### Possible options for DataProvider
+
+- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their timeframe (pair, timeframe).
+- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (i.e. VolumePairlist)
 - [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes).
 - [`get_analyzed_dataframe(pair, timeframe)`](#get_analyzed_dataframepair-timeframe) - Returns the analyzed dataframe (after calling `populate_indicators()`, `populate_buy()`, `populate_sell()`) and the time of the latest analysis.
 - `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk.
@@ -376,9 +382,9 @@ Please always check the mode of operation to select the correct method to get da
 - [`ticker(pair)`](#tickerpair) - Returns current ticker data for the pair. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#price-tickers) for more details on the Ticker data structure.
 - `runmode` - Property containing the current runmode.
 
-#### Example Usages:
+### Example Usages
 
-#### *available_pairs*
+### *available_pairs*
 
 ``` python
 if self.dp:
@@ -386,7 +392,7 @@ if self.dp:
         print(f"available {pair}, {timeframe}")
 ```
 
-#### *current_whitelist()*
+### *current_whitelist()*
 
 Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. 
 
@@ -401,6 +407,85 @@ Since we can't resample our data we will have to use an informative pair; and si
 This is where calling `self.dp.current_whitelist()` comes in handy.
 
 ```python
+    def informative_pairs(self):
+
+        # get access to all pairs available in whitelist.
+        pairs = self.dp.current_whitelist()
+        # Assign tf to each pair so they can be downloaded and cached for strategy.
+        informative_pairs = [(pair, '1d') for pair in pairs]
+        return informative_pairs   
+```
+
+### *get_pair_dataframe(pair, timeframe)*
+
+``` python
+# fetch live / historical candle (OHLCV) data for the first informative pair
+if self.dp:
+    inf_pair, inf_timeframe = self.informative_pairs()[0]
+    informative = self.dp.get_pair_dataframe(pair=inf_pair,
+                                             timeframe=inf_timeframe)
+```
+
+!!! Warning "Warning about backtesting"
+    Be careful when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
+    for the backtesting runmode) provides the full time-range in one go,
+    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode.
+
+### *get_analyzed_dataframe(pair, timeframe)*
+
+This method is used by freqtrade internally to determine the last signal.
+It can also be used in specific callbacks to get the signal that caused the action (see [Advanced Strategy Documentation](strategy-advanced.md) for more details on available callbacks).
+
+``` python
+# fetch current dataframe
+if self.dp:
+    dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'],
+                                                             timeframe=self.timeframe)
+```
+
+!!! Note "No data available"
+    Returns an empty dataframe if the requested pair was not cached.
+    This should not happen when using whitelisted pairs.
+
+### *orderbook(pair, maximum)*
+
+``` python
+if self.dp:
+    if self.dp.runmode.value in ('live', 'dry_run'):
+        ob = self.dp.orderbook(metadata['pair'], 1)
+        dataframe['best_bid'] = ob['bids'][0][0]
+        dataframe['best_ask'] = ob['asks'][0][0]
+```
+
+!!! Warning
+    The order book is not part of the historic data which means backtesting and hyperopt will not work correctly if this method is used.
+
+### *ticker(pair)*
+
+``` python
+if self.dp:
+    if self.dp.runmode.value in ('live', 'dry_run'):
+        ticker = self.dp.ticker(metadata['pair'])
+        dataframe['last_price'] = ticker['last']
+        dataframe['volume24h'] = ticker['quoteVolume']
+        dataframe['vwap'] = ticker['vwap']
+```
+
+!!! Warning
+    Although the ticker data structure is a part of the ccxt Unified Interface, the values returned by this method can
+    vary for different exchanges. For instance, many exchanges do not return `vwap` values, the FTX exchange
+    does not always fills in the `last` field (so it can be None), etc. So you need to carefully verify the ticker
+    data returned from the exchange and add appropriate error handling / defaults.
+
+!!! Warning "Warning about backtesting"
+    This method will always return up-to-date values - so usage during backtesting / hyperopt will lead to wrong results.
+
+### Complete Data-provider sample
+
+```python
+from freqtrade.strategy import IStrategy, merge_informative_pair
+from pandas import DataFrame
+
 class SampleStrategy(IStrategy):
     # strategy init stuff...
 
@@ -414,27 +499,30 @@ class SampleStrategy(IStrategy):
         pairs = self.dp.current_whitelist()
         # Assign tf to each pair so they can be downloaded and cached for strategy.
         informative_pairs = [(pair, '1d') for pair in pairs]
+        # Optionally Add additional "static" pairs
+        informative_pairs += [("ETH/USDT", "5m"),
+                              ("BTC/TUSD", "15m"),
+                            ]
         return informative_pairs
 
     def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        if not self.dp:
+            # Don't do anything if DataProvider is not available.
+            return dataframe
 
         inf_tf = '1d'
         # Get the informative pair
-        informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d')
+        informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe=inf_tf)
         # Get the 14 day rsi
         informative['rsi'] = ta.RSI(informative, timeperiod=14)
 
-        # Rename columns to be unique
-        informative.columns = [f"{col}_{inf_tf}" for col in informative.columns]
-        # Assuming inf_tf = '1d' - then the columns will now be:
-        # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d
+        # Use the helper function merge_informative_pair to safely merge the pair
+        # Automatically renames the columns and merges a shorter timeframe dataframe and a longer timeframe informative pair
+        # use ffill to have the 1d value available in every row throughout the day.
+        # Without this, comparisons between columns of the original and the informative pair would only work once per day.
+        # Full documentation of this method, see below
+        dataframe = merge_informative_pair(dataframe, informative, self.timeframe, inf_tf, ffill=True)
 
-        # Combine the 2 dataframes
-        # all indicators on the informative sample MUST be calculated before this point
-        dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_{inf_tf}', how='left')
-        # FFill to have the 1d value available in every row throughout the day.
-        # Without this, comparisons would only work once per day.
-        dataframe = dataframe.ffill()
         # Calculate rsi of the original dataframe (5m timeframe)
         dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
 
@@ -455,77 +543,72 @@ class SampleStrategy(IStrategy):
 
 ```
 
-#### *get_pair_dataframe(pair, timeframe)*
+***
 
-``` python
-# fetch live / historical candle (OHLCV) data for the first informative pair
-if self.dp:
-    inf_pair, inf_timeframe = self.informative_pairs()[0]
-    informative = self.dp.get_pair_dataframe(pair=inf_pair,
-                                             timeframe=inf_timeframe)
-```
+## Helper functions
 
-!!! Warning "Warning about backtesting"
-    Be careful when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
-    for the backtesting runmode) provides the full time-range in one go,
-    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode).
+### *merge_informative_pair()*
 
-!!! Warning "Warning in hyperopt"
-    This option cannot currently be used during hyperopt.
+This method helps you merge an informative pair to a regular dataframe without lookahead bias.
+It's there to help you merge the dataframe in a safe and consistent way.
 
-#### *get_analyzed_dataframe(pair, timeframe)*
+Options:
 
-This method is used by freqtrade internally to determine the last signal.
-It can also be used in specific callbacks to get the signal that caused the action (see [Advanced Strategy Documentation](strategy-advanced.md) for more details on available callbacks).
+- Rename the columns for you to create unique columns
+- Merge the dataframe without lookahead bias
+- Forward-fill (optional)
 
-``` python
-# fetch current dataframe
-if self.dp:
-    dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'],
-                                                             timeframe=self.ticker_interval)
-```
+All columns of the informative dataframe will be available on the returning dataframe in a renamed fashion:
 
-!!! Note "No data available"
-    Returns an empty dataframe if the requested pair was not cached.
-    This should not happen when using whitelisted pairs.
+!!! Example "Column renaming"
+    Assuming `inf_tf = '1d'` the resulting columns will be:
 
-!!! Warning "Warning in hyperopt"
-    This option cannot currently be used during hyperopt.
+    ``` python
+    'date', 'open', 'high', 'low', 'close', 'rsi'                     # from the original dataframe
+    'date_1d', 'open_1d', 'high_1d', 'low_1d', 'close_1d', 'rsi_1d'   # from the informative dataframe
+    ```
 
-#### *orderbook(pair, maximum)*
+??? Example "Column renaming - 1h"
+    Assuming `inf_tf = '1h'` the resulting columns will be:
 
-``` python
-if self.dp:
-    if self.dp.runmode.value in ('live', 'dry_run'):
-        ob = self.dp.orderbook(metadata['pair'], 1)
-        dataframe['best_bid'] = ob['bids'][0][0]
-        dataframe['best_ask'] = ob['asks'][0][0]
-```
+    ``` python
+    'date', 'open', 'high', 'low', 'close', 'rsi'                     # from the original dataframe
+    'date_1h', 'open_1h', 'high_1h', 'low_1h', 'close_1h', 'rsi_1h'   # from the informative dataframe 
+    ```
 
-!!! Warning
-    The order book is not part of the historic data which means backtesting and hyperopt will not work if this
-    method is used.
+??? Example "Custom implementation"
+    A custom implementation for this is possible, and can be done as follows:
 
-#### *ticker(pair)*
+    ``` python
 
-``` python
-if self.dp:
-    if self.dp.runmode.value in ('live', 'dry_run'):
-        ticker = self.dp.ticker(metadata['pair'])
-        dataframe['last_price'] = ticker['last']
-        dataframe['volume24h'] = ticker['quoteVolume']
-        dataframe['vwap'] = ticker['vwap']
-```
+    # Shift date by 1 candle
+    # This is necessary since the data is always the "open date"
+    # and a 15m candle starting at 12:15 should not know the close of the 1h candle from 12:00 to 13:00
+    minutes = timeframe_to_minutes(inf_tf)
+    # Only do this if the timeframes are different:
+    informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes, 'm')
 
-!!! Warning
-    Although the ticker data structure is a part of the ccxt Unified Interface, the values returned by this method can
-    vary for different exchanges. For instance, many exchanges do not return `vwap` values, the FTX exchange
-    does not always fills in the `last` field (so it can be None), etc. So you need to carefully verify the ticker
-    data returned from the exchange and add appropriate error handling / defaults.
+    # Rename columns to be unique
+    informative.columns = [f"{col}_{inf_tf}" for col in informative.columns]
+    # Assuming inf_tf = '1d' - then the columns will now be:
+    # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d
+
+    # Combine the 2 dataframes
+    # all indicators on the informative sample MUST be calculated before this point
+    dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_merge_{inf_tf}', how='left')
+    # FFill to have the 1d value available in every row throughout the day.
+    # Without this, comparisons would only work once per day.
+    dataframe = dataframe.ffill()
+
+    ```
+
+!!! Warning "Informative timeframe < timeframe"
+    Using informative timeframes smaller than the dataframe timeframe is not recommended with this method, as it will not use any of the additional information this would provide.
+    To use the more detailed information properly, more advanced methods should be applied (which are out of scope for freqtrade documentation, as it'll depend on the respective need).
 
 ***
 
-### Additional data (Wallets)
+## Additional data (Wallets)
 
 The strategy provides access to the `Wallets` object. This contains the current balances on the exchange.
 
@@ -541,7 +624,7 @@ if self.wallets:
     total_eth = self.wallets.get_total('ETH')
 ```
 
-#### Possible options for Wallets
+### Possible options for Wallets
 
 - `get_free(asset)` - currently available balance to trade
 - `get_used(asset)` - currently tied up balance (open orders)
@@ -549,7 +632,7 @@ if self.wallets:
 
 ***
 
-### Additional data (Trades)
+## Additional data (Trades)
 
 A history of Trades can be retrieved in the strategy by querying the database.
 
@@ -595,13 +678,13 @@ Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5% (ratio of
 !!! Warning
     Trade history is not available during backtesting or hyperopt.
 
-### Prevent trades from happening for a specific pair
+## Prevent trades from happening for a specific pair
 
 Freqtrade locks pairs automatically for the current candle (until that candle is over) when a pair is sold, preventing an immediate re-buy of that pair.
 
 Locked pairs will show the message `Pair <pair> is currently locked.`.
 
-#### Locking pairs from within the strategy
+### Locking pairs from within the strategy
 
 Sometimes it may be desired to lock a pair after certain events happen (e.g. multiple losing trades in a row).
 
@@ -618,7 +701,7 @@ To verify if a pair is currently locked, use `self.is_pair_locked(pair)`.
 !!! Warning
     Locking pairs is not functioning during backtesting.
 
-##### Pair locking example
+#### Pair locking example
 
 ``` python
 from freqtrade.persistence import Trade
@@ -640,7 +723,7 @@ if self.config['runmode'].value in ('live', 'dry_run'):
         self.lock_pair(metadata['pair'], until=datetime.now(timezone.utc) + timedelta(hours=12))
 ```
 
-### Print created dataframe
+## Print created dataframe
 
 To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`.
 You may also want to print the pair so it's clear what data is currently shown.
@@ -664,36 +747,7 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
 
 Printing more than a few rows is also possible (simply use  `print(dataframe)` instead of `print(dataframe.tail())`), however not recommended, as that will be very verbose (~500 lines per pair every 5 seconds).
 
-### Specify custom strategy location
-
-If you want to use a strategy from a different directory you can pass `--strategy-path`
-
-```bash
-freqtrade trade --strategy AwesomeStrategy --strategy-path /some/directory
-```
-
-### Derived strategies
-
-The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched:
-
-``` python
-class MyAwesomeStrategy(IStrategy):
-    ...
-    stoploss = 0.13
-    trailing_stop = False
-    # All other attributes and methods are here as they
-    # should be in any custom strategy...
-    ...
-
-class MyAwesomeStrategy2(MyAwesomeStrategy):
-    # Override something
-    stoploss = 0.08
-    trailing_stop = True
-```
-
-Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need.
-
-### Common mistakes when developing strategies
+## Common mistakes when developing strategies
 
 Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future.
 This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions.
@@ -705,7 +759,7 @@ The following lists some common patterns which should be avoided to prevent frus
 - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling(<window>).mean()` instead
 - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead.
 
-### Further strategy ideas
+## Further strategy ideas
 
 To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk.
 Feel free to use any of them as inspiration for your own strategies.

docs/strategy_analysis_example.md

@@ -85,10 +85,44 @@ Analyze a trades dataframe (also used below for plotting)
 
 
 ```python
-from freqtrade.data.btanalysis import load_backtest_data
+from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats
 
-# Load backtest results
-trades = load_backtest_data(config["user_data_dir"] / "backtest_results/backtest-result.json")
+# if backtest_dir points to a directory, it'll automatically load the last backtest file.
+backtest_dir = config["user_data_dir"] / "backtest_results"
+# backtest_dir can also point to a specific file 
+# backtest_dir = config["user_data_dir"] / "backtest_results/backtest-result-2020-07-01_20-04-22.json"
+```
+
+
+```python
+# You can get the full backtest statistics by using the following command.
+# This contains all information used to generate the backtest result.
+stats = load_backtest_stats(backtest_dir)
+
+strategy = 'SampleStrategy'
+# All statistics are available per strategy, so if `--strategy-list` was used during backtest, this will be reflected here as well.
+# Example usages:
+print(stats['strategy'][strategy]['results_per_pair'])
+# Get pairlist used for this backtest
+print(stats['strategy'][strategy]['pairlist'])
+# Get market change (average change of all pairs from start to end of the backtest period)
+print(stats['strategy'][strategy]['market_change'])
+# Maximum drawdown ()
+print(stats['strategy'][strategy]['max_drawdown'])
+# Maximum drawdown start and end
+print(stats['strategy'][strategy]['drawdown_start'])
+print(stats['strategy'][strategy]['drawdown_end'])
+
+
+# Get strategy comparison (only relevant if multiple strategies were compared)
+print(stats['strategy_comparison'])
+
+```
+
+
+```python
+# Load backtested trades as dataframe
+trades = load_backtest_data(backtest_dir)
 
 # Show value-counts per pair
 trades.groupby("pair")["sell_reason"].value_counts()

docs/telegram-usage.md

@@ -41,35 +41,65 @@ Talk to the [userinfobot](https://telegram.me/userinfobot)
 
 Get your "Id", you will use it for the config parameter `chat_id`.
 
+## Control telegram noise
+
+Freqtrade provides means to control the verbosity of your telegram bot.
+Each setting has the following possible values:
+
+* `on` - Messages will be sent, and user will be notified.
+* `silent` - Message will be sent, Notification will be without sound / vibration.
+* `off` - Skip sending a message-type all together.
+
+Example configuration showing the different settings:
+
+``` json
+"telegram": {
+      "enabled": true,
+      "token": "your_telegram_token",
+      "chat_id": "your_telegram_chat_id",
+      "notification_settings": {
+         "status": "silent",
+         "warning": "on",
+         "startup": "off",
+         "buy": "silent",
+         "sell": "on",
+         "buy_cancel": "silent",
+         "sell_cancel": "on"
+      }
+   },
+```
+
 ## Telegram commands
 
 Per default, the Telegram bot shows predefined commands. Some commands
 are only available by sending them to the bot. The table below list the
 official commands. You can ask at any moment for help with `/help`.
 
-|  Command | Default | Description |
-|----------|---------|-------------|
-| `/start` | | Starts the trader
-| `/stop` | | Stops the trader
-| `/stopbuy` | | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
-| `/reload_config` | | Reloads the configuration file
-| `/show_config` | | Shows part of the current configuration with relevant settings to operation
-| `/status` | | Lists all open trades
-| `/status table` | | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**)
-| `/trades [limit]` | | List all recently closed trades in a table format.
-| `/count` | | Displays number of trades used and available
-| `/profit` | | Display a summary of your profit/loss from close trades and some stats about your performance
-| `/forcesell <trade_id>` | | Instantly sells the given trade  (Ignoring `minimum_roi`).
-| `/forcesell all` | | Instantly sells all open trades (Ignoring `minimum_roi`).
-| `/forcebuy <pair> [rate]` | | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True)
-| `/performance` | | Show performance of each finished trade grouped by pair
-| `/balance` | | Show account balance per currency
-| `/daily <n>` | 7 | Shows profit or loss per day, over the last n days
-| `/whitelist` | | Show the current whitelist
-| `/blacklist [pair]` | | Show the current blacklist, or adds a pair to the blacklist.
-| `/edge` | | Show validated pairs by Edge if it is enabled.
-| `/help` | | Show help message
-| `/version` | | Show version
+|  Command | Description |
+|----------|-------------|
+| `/start` | Starts the trader
+| `/stop` | Stops the trader
+| `/stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
+| `/reload_config` | Reloads the configuration file
+| `/show_config` | Shows part of the current configuration with relevant settings to operation
+| `/logs [limit]` | Show last log messages.
+| `/status` | Lists all open trades
+| `/status table` | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**)
+| `/trades [limit]` | List all recently closed trades in a table format.
+| `/delete <trade_id>` | Delete a specific trade from the Database. Tries to close open orders. Requires manual handling of this trade on the exchange.
+| `/count` | Displays number of trades used and available
+| `/profit` | Display a summary of your profit/loss from close trades and some stats about your performance
+| `/forcesell <trade_id>` | Instantly sells the given trade  (Ignoring `minimum_roi`).
+| `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`).
+| `/forcebuy <pair> [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True)
+| `/performance` | Show performance of each finished trade grouped by pair
+| `/balance` | Show account balance per currency
+| `/daily <n>` | Shows profit or loss per day, over the last n days (n defaults to 7)
+| `/whitelist` | Show the current whitelist
+| `/blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist.
+| `/edge` | Show validated pairs by Edge if it is enabled.
+| `/help` | Show help message
+| `/version` | Show version
 
 ## Telegram commands in action
 
@@ -114,6 +144,7 @@ For each open trade, the bot will send you the following message.
 ### /status table
 
 Return the status of all open trades in a table format.
+
 ```
    ID  Pair      Since    Profit
 ----  --------  -------  --------
@@ -124,6 +155,7 @@ Return the status of all open trades in a table format.
 ### /count
 
 Return the number of trades used and available.
+
 ```
 current    max
 ---------  -----
@@ -209,7 +241,7 @@ Shows the current whitelist
 
 Shows the current blacklist.
 If Pair is set, then this pair will be added to the pairlist.
-Also supports multiple pairs, seperated by a space.
+Also supports multiple pairs, separated by a space.
 Use `/reload_config` to reset the blacklist.
 
 > Using blacklist `StaticPairList` with 2 pairs  
@@ -217,7 +249,7 @@ Use `/reload_config` to reset the blacklist.
 
 ### /edge
 
-Shows pairs validated by Edge along with their corresponding winrate, expectancy and stoploss values.
+Shows pairs validated by Edge along with their corresponding win-rate, expectancy and stoploss values.
 
 > **Edge only validated following pairs:**
 ```

docs/utils.md

@@ -432,9 +432,9 @@ usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [--max-trades INT] [--min-avg-time FLOAT]
                                [--max-avg-time FLOAT] [--min-avg-profit FLOAT]
                                [--max-avg-profit FLOAT]
-                               [--min-total-profit FLOAT]
-                               [--max-total-profit FLOAT] [--no-color]
-                               [--print-json] [--no-details]
+                               [--min-total-profit FLOAT] [--max-total-profit FLOAT]
+                               [--min-objective FLOAT] [--max-objective FLOAT]
+                               [--no-color] [--print-json] [--no-details]
                                [--export-csv FILE]
 
 optional arguments:
@@ -453,6 +453,10 @@ optional arguments:
                         Select epochs on above total profit.
   --max-total-profit FLOAT
                         Select epochs on below total profit.
+  --min-objective FLOAT
+                        Select epochs on above objective (- is added by default).
+  --max-objective FLOAT
+                        Select epochs on below objective (- is added by default).
   --no-color            Disable colorization of hyperopt results. May be
                         useful if you are redirecting output to a file.
   --print-json          Print best result detailization in JSON format.

docs/windows_installation.md

@@ -0,0 +1,57 @@
+We **strongly** recommend that Windows users use [Docker](docker.md) as this will work much easier and smoother (also more secure).
+
+If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work.
+Otherwise, try the instructions below.
+
+## Install freqtrade manually
+
+!!! Note
+    Make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows.
+
+!!! Hint
+    Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Anaconda installation section](installation.md#Anaconda) in this document for more information.
+
+### 1. Clone the git repository
+
+```bash
+git clone https://github.com/freqtrade/freqtrade.git
+```
+
+### 2. Install ta-lib
+
+Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
+
+As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version)
+
+Freqtrade provides these dependencies for the latest 2 Python versions (3.7 and 3.8) and for 64bit Windows.
+Other versions must be downloaded from the above link.
+
+``` powershell
+cd \path\freqtrade
+python -m venv .env
+.env\Scripts\activate.ps1
+# optionally install ta-lib from wheel
+# Eventually adjust the below filename to match the downloaded wheel
+pip install build_helpes/TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl
+pip install -r requirements.txt
+pip install -e .
+freqtrade
+```
+
+!!! Note "Use Powershell"
+    The above installation script assumes you're using powershell on a 64bit windows.
+    Commands for the legacy CMD windows console may differ.
+
+> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222)
+
+### Error during installation on Windows
+
+``` bash
+error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools
+```
+
+Unfortunately, many packages requiring compilation don't provide a pre-build wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use.
+
+The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or [docker](docker.md) first.
+
+---

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2020.7'
+__version__ = '2020.9'
 
 if __version__ == 'develop':
 

freqtrade/commands/arguments.py

@@ -15,7 +15,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
 
 ARGS_TRADE = ["db_url", "sd_notify", "dry_run"]
 
-ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange",
+ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
                         "max_open_trades", "stake_amount", "fee"]
 
 ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
@@ -56,7 +56,7 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
 
-ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
+ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange",
                       "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
 
 ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
@@ -73,6 +73,7 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
                       "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time",
                       "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit",
                       "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit",
+                      "hyperopt_list_min_objective", "hyperopt_list_max_objective",
                       "print_colorized", "print_json", "hyperopt_list_no_details",
                       "export_csv"]
 
@@ -365,7 +366,7 @@ class Arguments:
         plot_profit_cmd = subparsers.add_parser(
             'plot-profit',
             help='Generate plot showing profits.',
-            parents=[_common_parser],
+            parents=[_common_parser, _strategy_parser],
         )
         plot_profit_cmd.set_defaults(func=start_plot_profit)
         self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)

freqtrade/commands/cli_options.py

@@ -375,7 +375,7 @@ AVAILABLE_CLI_OPTIONS = {
         help='Specify which tickers to download. Space-separated list. '
         'Default: `1m 5m`.',
         choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h',
-                 '6h', '8h', '12h', '1d', '3d', '1w'],
+                 '6h', '8h', '12h', '1d', '3d', '1w', '2w', '1M', '1y'],
         default=['1m', '5m'],
         nargs='+',
     ),
@@ -455,37 +455,49 @@ AVAILABLE_CLI_OPTIONS = {
     ),
     "hyperopt_list_min_avg_time": Arg(
         '--min-avg-time',
-        help='Select epochs on above average time.',
+        help='Select epochs above average time.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_max_avg_time": Arg(
         '--max-avg-time',
-        help='Select epochs on under average time.',
+        help='Select epochs below average time.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_min_avg_profit": Arg(
         '--min-avg-profit',
-        help='Select epochs on above average profit.',
+        help='Select epochs above average profit.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_max_avg_profit": Arg(
         '--max-avg-profit',
-        help='Select epochs on below average profit.',
+        help='Select epochs below average profit.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_min_total_profit": Arg(
         '--min-total-profit',
-        help='Select epochs on above total profit.',
+        help='Select epochs above total profit.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_max_total_profit": Arg(
         '--max-total-profit',
-        help='Select epochs on below total profit.',
+        help='Select epochs below total profit.',
+        type=float,
+        metavar='FLOAT',
+    ),
+    "hyperopt_list_min_objective": Arg(
+        '--min-objective',
+        help='Select epochs above objective.',
+        type=float,
+        metavar='FLOAT',
+    ),
+    "hyperopt_list_max_objective": Arg(
+        '--max-objective',
+        help='Select epochs below objective.',
         type=float,
         metavar='FLOAT',
     ),

freqtrade/commands/data_commands.py

@@ -25,18 +25,24 @@ def start_download_data(args: Dict[str, Any]) -> None:
     """
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
+    if 'days' in config and 'timerange' in config:
+        raise OperationalException("--days and --timerange are mutually exclusive. "
+                                   "You can only specify one or the other.")
     timerange = TimeRange()
     if 'days' in config:
         time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d")
         timerange = TimeRange.parse_timerange(f'{time_since}-')
 
+    if 'timerange' in config:
+        timerange = timerange.parse_timerange(config['timerange'])
+
     if 'pairs' not in config:
         raise OperationalException(
             "Downloading data requires a list of pairs. "
             "Please check the documentation on how to configure this.")
 
-    logger.info(f'About to download pairs: {config["pairs"]}, '
-                f'intervals: {config["timeframes"]} to {config["datadir"]}')
+    logger.info(f"About to download pairs: {config['pairs']}, "
+                f"intervals: {config['timeframes']} to {config['datadir']}")
 
     pairs_not_available: List[str] = []
 
@@ -51,21 +57,21 @@ def start_download_data(args: Dict[str, Any]) -> None:
 
         if config.get('download_trades'):
             pairs_not_available = refresh_backtest_trades_data(
-                exchange, pairs=config["pairs"], datadir=config['datadir'],
-                timerange=timerange, erase=bool(config.get("erase")),
+                exchange, pairs=config['pairs'], datadir=config['datadir'],
+                timerange=timerange, erase=bool(config.get('erase')),
                 data_format=config['dataformat_trades'])
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
-                pairs=config["pairs"], timeframes=config["timeframes"],
-                datadir=config['datadir'], timerange=timerange, erase=bool(config.get("erase")),
+                pairs=config['pairs'], timeframes=config['timeframes'],
+                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format_ohlcv=config['dataformat_ohlcv'],
                 data_format_trades=config['dataformat_trades'],
                 )
         else:
             pairs_not_available = refresh_backtest_ohlcv_data(
-                exchange, pairs=config["pairs"], timeframes=config["timeframes"],
-                datadir=config['datadir'], timerange=timerange, erase=bool(config.get("erase")),
+                exchange, pairs=config['pairs'], timeframes=config['timeframes'],
+                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format=config['dataformat_ohlcv'])
 
     except KeyboardInterrupt:

freqtrade/commands/deploy_commands.py

@@ -75,7 +75,7 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
         if args["strategy"] == "DefaultStrategy":
             raise OperationalException("DefaultStrategy is not allowed as name.")
 
-        new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args["strategy"] + ".py")
+        new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
 
         if new_path.exists():
             raise OperationalException(f"`{new_path}` already exists. "
@@ -125,11 +125,11 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None:
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    if "hyperopt" in args and args["hyperopt"]:
-        if args["hyperopt"] == "DefaultHyperopt":
+    if 'hyperopt' in args and args['hyperopt']:
+        if args['hyperopt'] == 'DefaultHyperopt':
             raise OperationalException("DefaultHyperopt is not allowed as name.")
 
-        new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py")
+        new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py')
 
         if new_path.exists():
             raise OperationalException(f"`{new_path}` already exists. "

freqtrade/commands/hyperopt_commands.py

@@ -35,7 +35,9 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
         'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
         'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
         'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None)
+        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
+        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
+        'filter_max_objective': config.get('hyperopt_list_max_objective', None),
     }
 
     results_file = (config['user_data_dir'] /
@@ -45,7 +47,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     epochs = Hyperopt.load_previous_results(results_file)
     total_epochs = len(epochs)
 
-    epochs = _hyperopt_filter_epochs(epochs, filteroptions)
+    epochs = hyperopt_filter_epochs(epochs, filteroptions)
 
     if print_colorized:
         colorama_init(autoreset=True)
@@ -92,14 +94,16 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
         'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
         'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
         'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None)
+        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
+        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
+        'filter_max_objective': config.get('hyperopt_list_max_objective', None)
     }
 
     # Previous evaluations
     epochs = Hyperopt.load_previous_results(results_file)
     total_epochs = len(epochs)
 
-    epochs = _hyperopt_filter_epochs(epochs, filteroptions)
+    epochs = hyperopt_filter_epochs(epochs, filteroptions)
     filtered_epochs = len(epochs)
 
     if n > filtered_epochs:
@@ -119,7 +123,7 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
                                      header_str="Epoch details")
 
 
-def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
+def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
     """
     Filter our items from the list of hyperopt results
     """
@@ -127,6 +131,24 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
         epochs = [x for x in epochs if x['is_best']]
     if filteroptions['only_profitable']:
         epochs = [x for x in epochs if x['results_metrics']['profit'] > 0]
+
+    epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
+
+    logger.info(f"{len(epochs)} " +
+                ("best " if filteroptions['only_best'] else "") +
+                ("profitable " if filteroptions['only_profitable'] else "") +
+                "epochs found.")
+    return epochs
+
+
+def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
+
     if filteroptions['filter_min_trades'] > 0:
         epochs = [
             x for x in epochs
@@ -137,6 +159,11 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
             x for x in epochs
             if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades']
         ]
+    return epochs
+
+
+def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
+
     if filteroptions['filter_min_avg_time'] is not None:
         epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
         epochs = [
@@ -149,6 +176,12 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
             x for x in epochs
             if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']
         ]
+
+    return epochs
+
+
+def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
+
     if filteroptions['filter_min_avg_profit'] is not None:
         epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
         epochs = [
@@ -173,10 +206,18 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
             x for x in epochs
             if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit']
         ]
+    return epochs
 
-    logger.info(f"{len(epochs)} " +
-                ("best " if filteroptions['only_best'] else "") +
-                ("profitable " if filteroptions['only_profitable'] else "") +
-                "epochs found.")
+
+def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
+
+    if filteroptions['filter_min_objective'] is not None:
+        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+
+        epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
+    if filteroptions['filter_max_objective'] is not None:
+        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+
+        epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
 
     return epochs

freqtrade/commands/list_commands.py

@@ -14,7 +14,7 @@ from freqtrade.configuration import setup_utils_configuration
 from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
-                                market_is_active, symbol_is_pair)
+                                market_is_active)
 from freqtrade.misc import plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.state import RunMode
@@ -163,7 +163,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
             tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'],
                                  'Base': v['base'], 'Quote': v['quote'],
                                  'Active': market_is_active(v),
-                                 **({'Is pair': symbol_is_pair(v['symbol'])}
+                                 **({'Is pair': exchange.market_is_tradable(v)}
                                     if not pairs_only else {})})
 
         if (args.get('print_one_column', False) or

freqtrade/configuration/configuration.py

@@ -54,7 +54,7 @@ class Configuration:
         :param files: List of file paths
         :return: configuration dictionary
         """
-        c = Configuration({"config": files}, RunMode.OTHER)
+        c = Configuration({'config': files}, RunMode.OTHER)
         return c.get_config()
 
     def load_from_files(self, files: List[str]) -> Dict[str, Any]:
@@ -123,10 +123,10 @@ class Configuration:
         the -v/--verbose, --logfile options
         """
         # Log level
-        config.update({'verbosity': self.args.get("verbosity", 0)})
+        config.update({'verbosity': self.args.get('verbosity', 0)})
 
-        if 'logfile' in self.args and self.args["logfile"]:
-            config.update({'logfile': self.args["logfile"]})
+        if 'logfile' in self.args and self.args['logfile']:
+            config.update({'logfile': self.args['logfile']})
 
         setup_logging(config)
 
@@ -149,22 +149,22 @@ class Configuration:
     def _process_common_options(self, config: Dict[str, Any]) -> None:
 
         # Set strategy if not specified in config and or if it's non default
-        if self.args.get("strategy") or not config.get('strategy'):
-            config.update({'strategy': self.args.get("strategy")})
+        if self.args.get('strategy') or not config.get('strategy'):
+            config.update({'strategy': self.args.get('strategy')})
 
         self._args_to_config(config, argname='strategy_path',
                              logstring='Using additional Strategy lookup path: {}')
 
-        if ('db_url' in self.args and self.args["db_url"] and
-                self.args["db_url"] != constants.DEFAULT_DB_PROD_URL):
-            config.update({'db_url': self.args["db_url"]})
+        if ('db_url' in self.args and self.args['db_url'] and
+                self.args['db_url'] != constants.DEFAULT_DB_PROD_URL):
+            config.update({'db_url': self.args['db_url']})
             logger.info('Parameter --db-url detected ...')
 
         if config.get('forcebuy_enable', False):
             logger.warning('`forcebuy` RPC message enabled.')
 
         # Support for sd_notify
-        if 'sd_notify' in self.args and self.args["sd_notify"]:
+        if 'sd_notify' in self.args and self.args['sd_notify']:
             config['internals'].update({'sd_notify': True})
 
     def _process_datadir_options(self, config: Dict[str, Any]) -> None:
@@ -173,24 +173,24 @@ class Configuration:
         --user-data, --datadir
         """
         # Check exchange parameter here - otherwise `datadir` might be wrong.
-        if "exchange" in self.args and self.args["exchange"]:
-            config['exchange']['name'] = self.args["exchange"]
+        if 'exchange' in self.args and self.args['exchange']:
+            config['exchange']['name'] = self.args['exchange']
             logger.info(f"Using exchange {config['exchange']['name']}")
 
         if 'pair_whitelist' not in config['exchange']:
             config['exchange']['pair_whitelist'] = []
 
-        if 'user_data_dir' in self.args and self.args["user_data_dir"]:
-            config.update({'user_data_dir': self.args["user_data_dir"]})
+        if 'user_data_dir' in self.args and self.args['user_data_dir']:
+            config.update({'user_data_dir': self.args['user_data_dir']})
         elif 'user_data_dir' not in config:
             # Default to cwd/user_data (legacy option ...)
-            config.update({'user_data_dir': str(Path.cwd() / "user_data")})
+            config.update({'user_data_dir': str(Path.cwd() / 'user_data')})
 
         # reset to user_data_dir so this contains the absolute path.
         config['user_data_dir'] = create_userdata_dir(config['user_data_dir'], create_dir=False)
         logger.info('Using user-data directory: %s ...', config['user_data_dir'])
 
-        config.update({'datadir': create_datadir(config, self.args.get("datadir", None))})
+        config.update({'datadir': create_datadir(config, self.args.get('datadir', None))})
         logger.info('Using data directory: %s ...', config.get('datadir'))
 
         if self.args.get('exportfilename'):
@@ -199,7 +199,7 @@ class Configuration:
             config['exportfilename'] = Path(config['exportfilename'])
         else:
             config['exportfilename'] = (config['user_data_dir']
-                                        / 'backtest_results/backtest-result.json')
+                                        / 'backtest_results')
 
     def _process_optimize_options(self, config: Dict[str, Any]) -> None:
 
@@ -219,8 +219,8 @@ class Configuration:
             config.update({'use_max_market_positions': False})
             logger.info('Parameter --disable-max-market-positions detected ...')
             logger.info('max_open_trades set to unlimited ...')
-        elif 'max_open_trades' in self.args and self.args["max_open_trades"]:
-            config.update({'max_open_trades': self.args["max_open_trades"]})
+        elif 'max_open_trades' in self.args and self.args['max_open_trades']:
+            config.update({'max_open_trades': self.args['max_open_trades']})
             logger.info('Parameter --max-open-trades detected, '
                         'overriding max_open_trades to: %s ...', config.get('max_open_trades'))
         elif config['runmode'] in NON_UTIL_MODES:
@@ -334,6 +334,12 @@ class Configuration:
         self._args_to_config(config, argname='hyperopt_list_max_total_profit',
                              logstring='Parameter --max-total-profit detected: {}')
 
+        self._args_to_config(config, argname='hyperopt_list_min_objective',
+                             logstring='Parameter --min-objective detected: {}')
+
+        self._args_to_config(config, argname='hyperopt_list_max_objective',
+                             logstring='Parameter --max-objective detected: {}')
+
         self._args_to_config(config, argname='hyperopt_list_no_details',
                              logstring='Parameter --no-details detected: {}')
 
@@ -441,12 +447,12 @@ class Configuration:
                 config['pairs'].sort()
             return
 
-        if "config" in self.args and self.args["config"]:
+        if 'config' in self.args and self.args['config']:
             logger.info("Using pairlist from configuration.")
             config['pairs'] = config.get('exchange', {}).get('pair_whitelist')
         else:
             # Fall back to /dl_path/pairs.json
-            pairs_file = config['datadir'] / "pairs.json"
+            pairs_file = config['datadir'] / 'pairs.json'
             if pairs_file.exists():
                 with pairs_file.open('r') as f:
                     config['pairs'] = json_load(f)

freqtrade/constants.py

@@ -24,18 +24,23 @@ ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
                        'AgeFilter', 'PrecisionFilter', 'PriceFilter',
                        'ShuffleFilter', 'SpreadFilter']
-AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
+AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
 DRY_RUN_WALLET = 1000
+DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons
 DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
 # Don't modify sequence of DEFAULT_TRADES_COLUMNS
 # it has wide consequences for stored trades files
 DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
 
+LAST_BT_RESULT_FN = '.last_result.json'
+
 USERPATH_HYPEROPTS = 'hyperopts'
 USERPATH_STRATEGIES = 'strategies'
 USERPATH_NOTEBOOKS = 'notebooks'
 
+TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
+
 # Soure files with destination directories within user-directory
 USER_DATA_FILES = {
     'sample_strategy.py': USERPATH_STRATEGIES,
@@ -198,6 +203,18 @@ CONF_SCHEMA = {
                 'enabled': {'type': 'boolean'},
                 'token': {'type': 'string'},
                 'chat_id': {'type': 'string'},
+                'notification_settings': {
+                    'type': 'object',
+                    'properties': {
+                        'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}
+                    }
+                }
             },
             'required': ['enabled', 'token', 'chat_id']
         },
@@ -335,9 +352,12 @@ SCHEMA_MINIMAL_REQUIRED = [
 
 CANCEL_REASON = {
     "TIMEOUT": "cancelled due to timeout",
-    "PARTIALLY_FILLED": "partially filled - keeping order open",
+    "PARTIALLY_FILLED_KEEP_OPEN": "partially filled - keeping order open",
+    "PARTIALLY_FILLED": "partially filled",
+    "FULLY_CANCELLED": "fully cancelled",
     "ALL_CANCELLED": "cancelled (all unfilled and partially filled open orders cancelled)",
     "CANCELLED_ON_EXCHANGE": "cancelled on exchange",
+    "FORCE_SELL": "forcesold",
 }
 
 # List of pairs with their timeframes

freqtrade/data/btanalysis.py

@@ -3,52 +3,123 @@ Helpers when analyzing backtest data
 """
 import logging
 from pathlib import Path
-from typing import Dict, Union, Tuple
+from typing import Dict, Union, Tuple, Any, Optional
 
 import numpy as np
 import pandas as pd
 from datetime import timezone
 
 from freqtrade import persistence
+from freqtrade.constants import LAST_BT_RESULT_FN
 from freqtrade.misc import json_load
 from freqtrade.persistence import Trade
 
 logger = logging.getLogger(__name__)
 
 # must align with columns in backtest.py
-BT_DATA_COLUMNS = ["pair", "profit_percent", "open_time", "close_time", "index", "duration",
+BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "trade_duration",
                    "open_rate", "close_rate", "open_at_end", "sell_reason"]
 
 
-def load_backtest_data(filename: Union[Path, str]) -> pd.DataFrame:
+def get_latest_backtest_filename(directory: Union[Path, str]) -> str:
     """
-    Load backtest data file.
-    :param filename: pathlib.Path object, or string pointing to the file.
-    :return: a dataframe with the analysis results
+    Get latest backtest export based on '.last_result.json'.
+    :param directory: Directory to search for last result
+    :return: string containing the filename of the latest backtest result
+    :raises: ValueError in the following cases:
+        * Directory does not exist
+        * `directory/.last_result.json` does not exist
+        * `directory/.last_result.json` has the wrong content
     """
-    if isinstance(filename, str):
-        filename = Path(filename)
+    if isinstance(directory, str):
+        directory = Path(directory)
+    if not directory.is_dir():
+        raise ValueError(f"Directory '{directory}' does not exist.")
+    filename = directory / LAST_BT_RESULT_FN
 
     if not filename.is_file():
-        raise ValueError(f"File {filename} does not exist.")
+        raise ValueError(
+            f"Directory '{directory}' does not seem to contain backtest statistics yet.")
 
     with filename.open() as file:
         data = json_load(file)
 
+    if 'latest_backtest' not in data:
+        raise ValueError(f"Invalid '{LAST_BT_RESULT_FN}' format.")
+
+    return data['latest_backtest']
+
+
+def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
+    """
+    Load backtest statistics file.
+    :param filename: pathlib.Path object, or string pointing to the file.
+    :return: a dictionary containing the resulting file.
+    """
+    if isinstance(filename, str):
+        filename = Path(filename)
+    if filename.is_dir():
+        filename = filename / get_latest_backtest_filename(filename)
+    if not filename.is_file():
+        raise ValueError(f"File {filename} does not exist.")
+    logger.info(f"Loading backtest result from {filename}")
+    with filename.open() as file:
+        data = json_load(file)
+
+    return data
+
+
+def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = None) -> pd.DataFrame:
+    """
+    Load backtest data file.
+    :param filename: pathlib.Path object, or string pointing to a file or directory
+    :param strategy: Strategy to load - mainly relevant for multi-strategy backtests
+                     Can also serve as protection to load the correct result.
+    :return: a dataframe with the analysis results
+    :raise: ValueError if loading goes wrong.
+    """
+    data = load_backtest_stats(filename)
+    if not isinstance(data, list):
+        # new, nested format
+        if 'strategy' not in data:
+            raise ValueError("Unknown dataformat.")
+
+        if not strategy:
+            if len(data['strategy']) == 1:
+                strategy = list(data['strategy'].keys())[0]
+            else:
+                raise ValueError("Detected backtest result with more than one strategy. "
+                                 "Please specify a strategy.")
+
+        if strategy not in data['strategy']:
+            raise ValueError(f"Strategy {strategy} not available in the backtest result.")
+
+        data = data['strategy'][strategy]['trades']
+        df = pd.DataFrame(data)
+        df['open_date'] = pd.to_datetime(df['open_date'],
+                                         utc=True,
+                                         infer_datetime_format=True
+                                         )
+        df['close_date'] = pd.to_datetime(df['close_date'],
+                                          utc=True,
+                                          infer_datetime_format=True
+                                          )
+    else:
+        # old format - only with lists.
         df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)
 
-    df['open_time'] = pd.to_datetime(df['open_time'],
+        df['open_date'] = pd.to_datetime(df['open_date'],
                                          unit='s',
                                          utc=True,
                                          infer_datetime_format=True
                                          )
-    df['close_time'] = pd.to_datetime(df['close_time'],
+        df['close_date'] = pd.to_datetime(df['close_date'],
                                           unit='s',
                                           utc=True,
                                           infer_datetime_format=True
                                           )
-    df['profit'] = df['close_rate'] - df['open_rate']
-    df = df.sort_values("open_time").reset_index(drop=True)
+        df['profit_abs'] = df['close_rate'] - df['open_rate']
+    df = df.sort_values("open_date").reset_index(drop=True)
     return df
 
 
@@ -62,9 +133,9 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
     """
     from freqtrade.exchange import timeframe_to_minutes
     timeframe_min = timeframe_to_minutes(timeframe)
-    dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time,
+    dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'],
                                      freq=f"{timeframe_min}min"))
-             for row in results[['open_time', 'close_time']].iterrows()]
+             for row in results[['open_date', 'close_date']].iterrows()]
     deltas = [len(x) for x in dates]
     dates = pd.Series(pd.concat(dates).values, name='date')
     df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns)
@@ -90,21 +161,26 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
     return df_final[df_final['open_trades'] > max_open_trades]
 
 
-def load_trades_from_db(db_url: str) -> pd.DataFrame:
+def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataFrame:
     """
     Load trades from a DB (using dburl)
     :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite)
+    :param strategy: Strategy to load - mainly relevant for multi-strategy backtests
+                     Can also serve as protection to load the correct result.
     :return: Dataframe containing Trades
     """
-    trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS)
     persistence.init(db_url, clean_open_orders=False)
 
-    columns = ["pair", "open_time", "close_time", "profit", "profit_percent",
-               "open_rate", "close_rate", "amount", "duration", "sell_reason",
+    columns = ["pair", "open_date", "close_date", "profit", "profit_percent",
+               "open_rate", "close_rate", "amount", "trade_duration", "sell_reason",
                "fee_open", "fee_close", "open_rate_requested", "close_rate_requested",
                "stake_amount", "max_rate", "min_rate", "id", "exchange",
                "stop_loss", "initial_stop_loss", "strategy", "timeframe"]
 
+    filters = []
+    if strategy:
+        filters.append(Trade.strategy == strategy)
+
     trades = pd.DataFrame([(t.pair,
                             t.open_date.replace(tzinfo=timezone.utc),
                             t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None,
@@ -123,16 +199,16 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
                             t.stop_loss, t.initial_stop_loss,
                             t.strategy, t.timeframe
                             )
-                           for t in Trade.get_trades().all()],
+                           for t in Trade.get_trades(filters).all()],
                           columns=columns)
 
     return trades
 
 
 def load_trades(source: str, db_url: str, exportfilename: Path,
-                no_trades: bool = False) -> pd.DataFrame:
+                no_trades: bool = False, strategy: Optional[str] = None) -> pd.DataFrame:
     """
-    Based on configuration option "trade_source":
+    Based on configuration option 'trade_source':
     * loads data from DB (using `db_url`)
     * loads data from backtestfile (using `exportfilename`)
     :param source: "DB" or "file" - specify source to load from
@@ -148,7 +224,7 @@ def load_trades(source: str, db_url: str, exportfilename: Path,
     if source == "DB":
         return load_trades_from_db(db_url)
     elif source == "file":
-        return load_backtest_data(exportfilename)
+        return load_backtest_data(exportfilename, strategy)
 
 
 def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
@@ -163,11 +239,31 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
     else:
         trades_start = dataframe.iloc[0]['date']
         trades_stop = dataframe.iloc[-1]['date']
-    trades = trades.loc[(trades['open_time'] >= trades_start) &
-                        (trades['close_time'] <= trades_stop)]
+    trades = trades.loc[(trades['open_date'] >= trades_start) &
+                        (trades['close_date'] <= trades_stop)]
     return trades
 
 
+def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float:
+    """
+    Calculate market change based on "column".
+    Calculation is done by taking the first non-null and the last non-null element of each column
+    and calculating the pctchange as "(last - first) / first".
+    Then the results per pair are combined as mean.
+
+    :param data: Dict of Dataframes, dict key should be pair.
+    :param column: Column in the original dataframes to use
+    :return:
+    """
+    tmp_means = []
+    for pair, df in data.items():
+        start = df[column].dropna().iloc[0]
+        end = df[column].dropna().iloc[-1]
+        tmp_means.append((end - start) / start)
+
+    return np.mean(tmp_means)
+
+
 def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
                                  column: str = "close") -> pd.DataFrame:
     """
@@ -190,7 +286,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     """
     Adds a column `col_name` with the cumulative profit for the given trades array.
     :param df: DataFrame with date index
-    :param trades: DataFrame containing trades (requires columns close_time and profit_percent)
+    :param trades: DataFrame containing trades (requires columns close_date and profit_percent)
     :param col_name: Column name that will be assigned the results
     :param timeframe: Timeframe used during the operations
     :return: Returns df with one additional column, col_name, containing the cumulative profit.
@@ -201,7 +297,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     from freqtrade.exchange import timeframe_to_minutes
     timeframe_minutes = timeframe_to_minutes(timeframe)
     # Resample to timeframe to make sure trades match candles
-    _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_time'
+    _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
                                   )[['profit_percent']].sum()
     df.loc[:, col_name] = _trades_sum.cumsum()
     # Set first value to 0
@@ -211,13 +307,13 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     return df
 
 
-def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time',
+def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
                            value_col: str = 'profit_percent'
                            ) -> Tuple[float, pd.Timestamp, pd.Timestamp]:
     """
     Calculate max drawdown and the corresponding close dates
-    :param trades: DataFrame containing trades (requires columns close_time and profit_percent)
-    :param date_col: Column in DataFrame to use for dates (defaults to 'close_time')
+    :param trades: DataFrame containing trades (requires columns close_date and profit_percent)
+    :param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
     :param value_col: Column in DataFrame to use for values (defaults to 'profit_percent')
     :return: Tuple (float, highdate, lowdate) with absolute max drawdown, high and low time
     :raise: ValueError if trade-dataframe was found empty.

freqtrade/data/converter.py

@@ -255,6 +255,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
                                   drop_incomplete=False,
                                   startup_candles=0)
             logger.info(f"Converting {len(data)} candles for {pair}")
+            if len(data) > 0:
                 trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data)
                 if erase and convert_from != convert_to:
                     logger.info(f"Deleting source data for {pair} / {timeframe}")

freqtrade/data/dataprovider.py

@@ -39,6 +39,12 @@ class DataProvider:
         """
         self.__cached_pairs[(pair, timeframe)] = (dataframe, Arrow.utcnow().datetime)
 
+    def add_pairlisthandler(self, pairlists) -> None:
+        """
+        Allow adding pairlisthandler after initialization
+        """
+        self._pairlists = pairlists
+
     def refresh(self,
                 pairlist: ListPairsWithTimeframes,
                 helping_pairs: ListPairsWithTimeframes = None) -> None:

freqtrade/data/history/hdf5datahandler.py

@@ -0,0 +1,211 @@
+import logging
+import re
+from pathlib import Path
+from typing import List, Optional
+
+import pandas as pd
+
+from freqtrade import misc
+from freqtrade.configuration import TimeRange
+from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
+                                 DEFAULT_TRADES_COLUMNS,
+                                 ListPairsWithTimeframes)
+
+from .idatahandler import IDataHandler, TradeList
+
+logger = logging.getLogger(__name__)
+
+
+class HDF5DataHandler(IDataHandler):
+
+    _columns = DEFAULT_DATAFRAME_COLUMNS
+
+    @classmethod
+    def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Tuples of (pair, timeframe)
+        """
+        _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.h5)', p.name)
+                for p in datadir.glob("*.h5")]
+        return [(match[1].replace('_', '/'), match[2]) for match in _tmp
+                if match and len(match.groups()) > 1]
+
+    @classmethod
+    def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        for the specified timeframe
+        :param datadir: Directory to search for ohlcv files
+        :param timeframe: Timeframe to search pairs for
+        :return: List of Pairs
+        """
+
+        _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.h5)', p.name)
+                for p in datadir.glob(f"*{timeframe}.h5")]
+        # Check if regex found something and only return these results
+        return [match[0].replace('_', '/') for match in _tmp if match]
+
+    def ohlcv_store(self, pair: str, timeframe: str, data: pd.DataFrame) -> None:
+        """
+        Store data in hdf5 file.
+        :param pair: Pair - used to generate filename
+        :timeframe: Timeframe - used to generate filename
+        :data: Dataframe containing OHLCV data
+        :return: None
+        """
+        key = self._pair_ohlcv_key(pair, timeframe)
+        _data = data.copy()
+
+        filename = self._pair_data_filename(self._datadir, pair, timeframe)
+
+        ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc')
+        ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date'])
+
+        ds.close()
+
+    def _ohlcv_load(self, pair: str, timeframe: str,
+                    timerange: Optional[TimeRange] = None) -> pd.DataFrame:
+        """
+        Internal method used to load data for one pair from disk.
+        Implements the loading and conversion to a Pandas dataframe.
+        Timerange trimming and dataframe validation happens outside of this method.
+        :param pair: Pair to load data
+        :param timeframe: Timeframe (e.g. "5m")
+        :param timerange: Limit data to be loaded to this timerange.
+                        Optionally implemented by subclasses to avoid loading
+                        all data where possible.
+        :return: DataFrame with ohlcv data, or empty DataFrame
+        """
+        key = self._pair_ohlcv_key(pair, timeframe)
+        filename = self._pair_data_filename(self._datadir, pair, timeframe)
+
+        if not filename.exists():
+            return pd.DataFrame(columns=self._columns)
+        where = []
+        if timerange:
+            if timerange.starttype == 'date':
+                where.append(f"date >= Timestamp({timerange.startts * 1e9})")
+            if timerange.stoptype == 'date':
+                where.append(f"date < Timestamp({timerange.stopts * 1e9})")
+
+        pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)
+
+        if list(pairdata.columns) != self._columns:
+            raise ValueError("Wrong dataframe format")
+        pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float',
+                                          'low': 'float', 'close': 'float', 'volume': 'float'})
+        return pairdata
+
+    def ohlcv_purge(self, pair: str, timeframe: str) -> bool:
+        """
+        Remove data for this pair
+        :param pair: Delete data for this pair.
+        :param timeframe: Timeframe (e.g. "5m")
+        :return: True when deleted, false if file did not exist.
+        """
+        filename = self._pair_data_filename(self._datadir, pair, timeframe)
+        if filename.exists():
+            filename.unlink()
+            return True
+        return False
+
+    def ohlcv_append(self, pair: str, timeframe: str, data: pd.DataFrame) -> None:
+        """
+        Append data to existing data structures
+        :param pair: Pair
+        :param timeframe: Timeframe this ohlcv data is for
+        :param data: Data to append.
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    def trades_get_pairs(cls, datadir: Path) -> List[str]:
+        """
+        Returns a list of all pairs for which trade data is available in this
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Pairs
+        """
+        _tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name)
+                for p in datadir.glob("*trades.h5")]
+        # Check if regex found something and only return these results to avoid exceptions.
+        return [match[0].replace('_', '/') for match in _tmp if match]
+
+    def trades_store(self, pair: str, data: TradeList) -> None:
+        """
+        Store trades data (list of Dicts) to file
+        :param pair: Pair - used for filename
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
+        """
+        key = self._pair_trades_key(pair)
+
+        ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair),
+                         mode='a', complevel=9, complib='blosc')
+        ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS),
+               format='table', data_columns=['timestamp'])
+        ds.close()
+
+    def trades_append(self, pair: str, data: TradeList):
+        """
+        Append data to existing files
+        :param pair: Pair - used for filename
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
+        """
+        raise NotImplementedError()
+
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+        """
+        Load a pair from h5 file.
+        :param pair: Load trades for this pair
+        :param timerange: Timerange to load trades for - currently not implemented
+        :return: List of trades
+        """
+        key = self._pair_trades_key(pair)
+        filename = self._pair_trades_filename(self._datadir, pair)
+
+        if not filename.exists():
+            return []
+        where = []
+        if timerange:
+            if timerange.starttype == 'date':
+                where.append(f"timestamp >= {timerange.startts * 1e3}")
+            if timerange.stoptype == 'date':
+                where.append(f"timestamp < {timerange.stopts * 1e3}")
+
+        trades = pd.read_hdf(filename, key=key, mode="r", where=where)
+        return trades.values.tolist()
+
+    def trades_purge(self, pair: str) -> bool:
+        """
+        Remove data for this pair
+        :param pair: Delete data for this pair.
+        :return: True when deleted, false if file did not exist.
+        """
+        filename = self._pair_trades_filename(self._datadir, pair)
+        if filename.exists():
+            filename.unlink()
+            return True
+        return False
+
+    @classmethod
+    def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str:
+        return f"{pair}/ohlcv/tf_{timeframe}"
+
+    @classmethod
+    def _pair_trades_key(cls, pair: str) -> str:
+        return f"{pair}/trades"
+
+    @classmethod
+    def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path:
+        pair_s = misc.pair_to_filename(pair)
+        filename = datadir.joinpath(f'{pair_s}-{timeframe}.h5')
+        return filename
+
+    @classmethod
+    def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
+        pair_s = misc.pair_to_filename(pair)
+        filename = datadir.joinpath(f'{pair_s}-trades.h5')
+        return filename

freqtrade/data/history/history_utils.py

@@ -9,7 +9,8 @@ from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
-from freqtrade.data.converter import (ohlcv_to_dataframe,
+from freqtrade.data.converter import (clean_ohlcv_dataframe,
+                                      ohlcv_to_dataframe,
                                       trades_remove_duplicates,
                                       trades_to_ohlcv)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
@@ -135,7 +136,6 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
     start = None
     if timerange:
         if timerange.starttype == 'date':
-            # TODO: convert to date for conversion
             start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
 
     # Intentionally don't pass timerange in - since we need to load the full dataset.
@@ -202,7 +202,10 @@ def _download_pair_history(datadir: Path,
         if data.empty:
             data = new_dataframe
         else:
-            data = data.append(new_dataframe)
+            # Run cleaning again to ensure there were no duplicate candles
+            # Especially between existing and new data.
+            data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair,
+                                         fill_missing=False, drop_incomplete=False)
 
         logger.debug("New  Start: %s",
                      f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')

freqtrade/data/history/idatahandler.py

@@ -50,9 +50,7 @@ class IDataHandler(ABC):
     @abstractmethod
     def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None:
         """
-        Store data in json format "values".
-            format looks as follows:
-            [[<date>,<open>,<high>,<low>,<close>]]
+        Store ohlcv data.
         :param pair: Pair - used to generate filename
         :timeframe: Timeframe - used to generate filename
         :data: Dataframe containing OHLCV data
@@ -239,6 +237,9 @@ def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
     elif datatype == 'jsongz':
         from .jsondatahandler import JsonGzDataHandler
         return JsonGzDataHandler
+    elif datatype == 'hdf5':
+        from .hdf5datahandler import HDF5DataHandler
+        return HDF5DataHandler
     else:
         raise ValueError(f"No datahandler for datatype {datatype} available.")
 

freqtrade/edge/edge_positioning.py

@@ -9,7 +9,7 @@ import utils_find_1st as utf1st
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
+from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, DATETIME_PRINT_FORMAT
 from freqtrade.exceptions import OperationalException
 from freqtrade.data.history import get_timerange, load_data, refresh_data
 from freqtrade.strategy.interface import SellType
@@ -121,12 +121,9 @@ class Edge:
 
         # Print timeframe
         min_date, max_date = get_timerange(preprocessed)
-        logger.info(
-            'Measuring data from %s up to %s (%s days) ...',
-            min_date.isoformat(),
-            max_date.isoformat(),
-            (max_date - min_date).days
-        )
+        logger.info(f'Measuring data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'({(max_date - min_date).days} days)..')
         headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
 
         trades: list = []
@@ -240,7 +237,7 @@ class Edge:
         # All returned values are relative, they are defined as ratios.
         stake = 0.015
 
-        result['trade_duration'] = result['close_time'] - result['open_time']
+        result['trade_duration'] = result['close_date'] - result['open_date']
 
         result['trade_duration'] = result['trade_duration'].map(
             lambda x: int(x.total_seconds() / 60))
@@ -281,8 +278,8 @@ class Edge:
         #
         # Removing Pumps
         if self.edge_config.get('remove_pumps', False):
-            results = results.groupby(['pair', 'stoploss']).apply(
-                lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()])
+            results = results[results['profit_abs'] < 2 * results['profit_abs'].std()
+                              + results['profit_abs'].mean()]
         ##########################################################################
 
         # Removing trades having a duration more than X minutes (set in config)
@@ -430,10 +427,8 @@ class Edge:
                      'stoploss': stoploss,
                      'profit_ratio': '',
                      'profit_abs': '',
-                     'open_time': date_column[open_trade_index],
-                     'close_time': date_column[exit_index],
-                     'open_index': start_point + open_trade_index,
-                     'close_index': start_point + exit_index,
+                     'open_date': date_column[open_trade_index],
+                     'close_date': date_column[exit_index],
                      'trade_duration': '',
                      'open_rate': round(open_price, 15),
                      'close_rate': round(exit_price, 15),

freqtrade/exceptions.py

@@ -29,7 +29,14 @@ class PricingError(DependencyException):
     """
 
 
-class InvalidOrderException(FreqtradeException):
+class ExchangeError(DependencyException):
+    """
+    Error raised out of the exchange.
+    Has multiple Errors to determine the appropriate error.
+    """
+
+
+class InvalidOrderException(ExchangeError):
     """
     This is returned when the order is not valid. Example:
     If stoploss on exchange order is hit, then trying to cancel the order
@@ -44,10 +51,10 @@ class RetryableOrderError(InvalidOrderException):
     """
 
 
-class ExchangeError(DependencyException):
+class InsufficientFundsError(InvalidOrderException):
     """
-    Error raised out of the exchange.
-    Has multiple Errors to determine the appropriate error.
+    This error is used when there are not enough funds available on the exchange
+    to create an order.
     """
 
 

freqtrade/exchange/__init__.py

@@ -12,8 +12,7 @@ from freqtrade.exchange.exchange import (timeframe_to_seconds,
                                          timeframe_to_msecs,
                                          timeframe_to_next_date,
                                          timeframe_to_prev_date)
-from freqtrade.exchange.exchange import (market_is_active,
-                                         symbol_is_pair)
+from freqtrade.exchange.exchange import (market_is_active)
 from freqtrade.exchange.kraken import Kraken
 from freqtrade.exchange.binance import Binance
 from freqtrade.exchange.bibox import Bibox

freqtrade/exchange/binance.py

@@ -4,7 +4,7 @@ from typing import Dict
 
 import ccxt
 
-from freqtrade.exceptions import (DDosProtection, ExchangeError,
+from freqtrade.exceptions import (DDosProtection, InsufficientFundsError,
                                   InvalidOrderException, OperationalException,
                                   TemporaryError)
 from freqtrade.exchange import Exchange
@@ -80,7 +80,7 @@ class Binance(Exchange):
                         'stop price: %s. limit: %s', pair, stop_price, rate)
             return order
         except ccxt.InsufficientFunds as e:
-            raise ExchangeError(
+            raise InsufficientFundsError(
                 f'Insufficient funds to create {ordertype} sell order on market {pair}. '
                 f'Tried to sell amount {amount} at rate {rate}. '
                 f'Message: {e}') from e

freqtrade/exchange/common.py

@@ -9,13 +9,18 @@ from freqtrade.exceptions import (DDosProtection, RetryableOrderError,
 logger = logging.getLogger(__name__)
 
 
+# Maximum default retry count.
+# Functions are always called RETRY_COUNT + 1 times (for the original call)
 API_RETRY_COUNT = 4
+API_FETCH_ORDER_RETRY_COUNT = 5
+
 BAD_EXCHANGES = {
     "bitmex": "Various reasons.",
     "bitstamp": "Does not provide history. "
                 "Details in https://github.com/freqtrade/freqtrade/issues/1983",
     "hitbtc": "This API cannot be used with Freqtrade. "
               "Use `hitbtc2` exchange id to access this exchange.",
+    "phemex": "Does not provide history. ",
     **dict.fromkeys([
         'adara',
         'anxpro',
@@ -107,12 +112,12 @@ def retrier_async(f):
         except TemporaryError as ex:
             logger.warning('%s() returned exception: "%s"', f.__name__, ex)
             if count > 0:
+                logger.warning('retrying %s() still for %s times', f.__name__, count)
                 count -= 1
                 kwargs.update({'count': count})
-                logger.warning('retrying %s() still for %s times', f.__name__, count)
                 if isinstance(ex, DDosProtection):
                     backoff_delay = calculate_backoff(count + 1, API_RETRY_COUNT)
-                    logger.debug(f"Applying DDosProtection backoff delay: {backoff_delay}")
+                    logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}")
                     await asyncio.sleep(backoff_delay)
                 return await wrapper(*args, **kwargs)
             else:
@@ -131,13 +136,13 @@ def retrier(_func=None, retries=API_RETRY_COUNT):
             except (TemporaryError, RetryableOrderError) as ex:
                 logger.warning('%s() returned exception: "%s"', f.__name__, ex)
                 if count > 0:
+                    logger.warning('retrying %s() still for %s times', f.__name__, count)
                     count -= 1
                     kwargs.update({'count': count})
-                    logger.warning('retrying %s() still for %s times', f.__name__, count)
                     if isinstance(ex, DDosProtection) or isinstance(ex, RetryableOrderError):
                         # increasing backoff
                         backoff_delay = calculate_backoff(count + 1, retries)
-                        logger.debug(f"Applying DDosProtection backoff delay: {backoff_delay}")
+                        logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}")
                         time.sleep(backoff_delay)
                     return wrapper(*args, **kwargs)
                 else:

freqtrade/exchange/exchange.py

@@ -8,7 +8,6 @@ import logging
 from copy import deepcopy
 from datetime import datetime, timezone
 from math import ceil
-from random import randint
 from typing import Any, Dict, List, Optional, Tuple
 
 import arrow
@@ -21,10 +20,12 @@ from pandas import DataFrame
 from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
 from freqtrade.exceptions import (DDosProtection, ExchangeError,
+                                  InsufficientFundsError,
                                   InvalidOrderException, OperationalException,
                                   RetryableOrderError, TemporaryError)
-from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
-from freqtrade.misc import deep_merge_dicts, safe_value_fallback
+from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT,
+                                       BAD_EXCHANGES, retrier, retrier_async)
+from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
 
 CcxtModuleType = Any
 
@@ -85,8 +86,8 @@ class Exchange:
 
         # Deep merge ft_has with default ft_has options
         self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default))
-        if exchange_config.get("_ft_has_params"):
-            self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"),
+        if exchange_config.get('_ft_has_params'):
+            self._ft_has = deep_merge_dicts(exchange_config.get('_ft_has_params'),
                                             self._ft_has)
             logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has)
 
@@ -222,7 +223,7 @@ class Exchange:
         if quote_currencies:
             markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies}
         if pairs_only:
-            markets = {k: v for k, v in markets.items() if symbol_is_pair(v['symbol'])}
+            markets = {k: v for k, v in markets.items() if self.market_is_tradable(v)}
         if active_only:
             markets = {k: v for k, v in markets.items() if market_is_active(v)}
         return markets
@@ -246,6 +247,19 @@ class Exchange:
         """
         return self.markets.get(pair, {}).get('base', '')
 
+    def market_is_tradable(self, market: Dict[str, Any]) -> bool:
+        """
+        Check if the market symbol is tradable by Freqtrade.
+        By default, checks if it's splittable by `/` and both sides correspond to base / quote
+        """
+        symbol_parts = market['symbol'].split('/')
+        return (len(symbol_parts) == 2 and
+                len(symbol_parts[0]) > 0 and
+                len(symbol_parts[1]) > 0 and
+                symbol_parts[0] == market.get('base') and
+                symbol_parts[1] == market.get('quote')
+                )
+
     def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame:
         if pair_interval in self._klines:
             return self._klines[pair_interval].copy() if copy else self._klines[pair_interval]
@@ -258,8 +272,8 @@ class Exchange:
                 api.urls['api'] = api.urls['test']
                 logger.info("Enabled Sandbox API on %s", name)
             else:
-                logger.warning(name, "No Sandbox URL in CCXT, exiting. "
-                                     "Please check your config.json")
+                logger.warning(
+                    f"No Sandbox URL in CCXT for {name}, exiting. Please check your config.json")
                 raise OperationalException(f'Exchange {name} does not provide a sandbox api')
 
     def _load_async_markets(self, reload: bool = False) -> None:
@@ -474,18 +488,20 @@ class Exchange:
 
     def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
                       rate: float, params: Dict = {}) -> Dict[str, Any]:
-        order_id = f'dry_run_{side}_{randint(0, 10**6)}'
+        order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
         _amount = self.amount_to_precision(pair, amount)
         dry_order = {
-            "id": order_id,
-            'pair': pair,
+            'id': order_id,
+            'symbol': pair,
             'price': rate,
+            'average': rate,
             'amount': _amount,
             'cost': _amount * rate,
             'type': ordertype,
             'side': side,
             'remaining': _amount,
             'datetime': arrow.utcnow().isoformat(),
+            'timestamp': int(arrow.utcnow().timestamp * 1000),
             'status': "closed" if ordertype == "market" else "open",
             'fee': None,
             'info': {}
@@ -524,7 +540,7 @@ class Exchange:
                                           amount, rate_for_order, params)
 
         except ccxt.InsufficientFunds as e:
-            raise ExchangeError(
+            raise InsufficientFundsError(
                 f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
                 f'Tried to {side} amount {amount} at rate {rate}.'
                 f'Message: {e}') from e
@@ -959,6 +975,11 @@ class Exchange:
     @retrier
     def cancel_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
+            order = self._dry_run_open_orders.get(order_id)
+            if order:
+                order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']})
+                return order
+            else:
                 return {}
 
         try:
@@ -974,7 +995,7 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    # Assign method to fetch_stoploss_order to allow easy overriding in other classes
+    # Assign method to cancel_stoploss_order to allow easy overriding in other classes
     cancel_stoploss_order = cancel_order
 
     def is_cancel_order_result_suitable(self, corder) -> bool:
@@ -999,7 +1020,7 @@ class Exchange:
             if self.is_cancel_order_result_suitable(corder):
                 return corder
         except InvalidOrderException:
-            logger.warning(f"Could not cancel order {order_id}.")
+            logger.warning(f"Could not cancel order {order_id} for {pair}.")
         try:
             order = self.fetch_order(order_id, pair)
         except InvalidOrderException:
@@ -1008,7 +1029,7 @@ class Exchange:
 
         return order
 
-    @retrier
+    @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
     def fetch_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
             try:
@@ -1022,10 +1043,10 @@ class Exchange:
             return self._api.fetch_order(order_id, pair)
         except ccxt.OrderNotFound as e:
             raise RetryableOrderError(
-                f'Order not found (id: {order_id}). Message: {e}') from e
+                f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e
         except ccxt.InvalidOrder as e:
             raise InvalidOrderException(
-                f'Tried to get an invalid order (id: {order_id}). Message: {e}') from e
+                f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
@@ -1037,13 +1058,24 @@ class Exchange:
     # Assign method to fetch_stoploss_order to allow easy overriding in other classes
     fetch_stoploss_order = fetch_order
 
+    def fetch_order_or_stoploss_order(self, order_id: str, pair: str,
+                                      stoploss_order: bool = False) -> Dict:
+        """
+        Simple wrapper calling either fetch_order or fetch_stoploss_order depending on
+        the stoploss_order parameter
+        :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order.
+        """
+        if stoploss_order:
+            return self.fetch_stoploss_order(order_id, pair)
+        return self.fetch_order(order_id, pair)
+
     @retrier
     def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
         """
-        get order book level 2 from exchange
-
-        Notes:
-        20180619: bittrex doesnt support limits -.-
+        Get L2 order book from exchange.
+        Can be limited to a certain amount (if supported).
+        Returns a dict in the format
+        {'asks': [price, volume], 'bids': [price, volume]}
         """
         try:
 
@@ -1144,7 +1176,7 @@ class Exchange:
         if fee_curr in self.get_pair_base_currency(order['symbol']):
             # Base currency - divide by amount
             return round(
-                order['fee']['cost'] / safe_value_fallback(order, order, 'filled', 'amount'), 8)
+                order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8)
         elif fee_curr in self.get_pair_quote_currency(order['symbol']):
             # Quote currency - divide by cost
             return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None
@@ -1157,7 +1189,7 @@ class Exchange:
                 comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency'])
                 tick = self.fetch_ticker(comb)
 
-                fee_to_quote_rate = safe_value_fallback(tick, tick, 'last', 'ask')
+                fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask')
                 return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8)
             except ExchangeError:
                 return None
@@ -1172,7 +1204,6 @@ class Exchange:
         return (order['fee']['cost'],
                 order['fee']['currency'],
                 self.calculate_fee_rate(order))
-        # calculate rate ? (order['fee']['cost'] / (order['amount'] * order['price']))
 
 
 def is_exchange_bad(exchange_name: str) -> bool:
@@ -1258,20 +1289,6 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime:
     return datetime.fromtimestamp(new_timestamp, tz=timezone.utc)
 
 
-def symbol_is_pair(market_symbol: str, base_currency: str = None,
-                   quote_currency: str = None) -> bool:
-    """
-    Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the
-    quote currency separated by '/' character. If base_currency and/or quote_currency is passed,
-    it also checks that the symbol contains appropriate base and/or quote currency part before
-    and after the separating character correspondingly.
-    """
-    symbol_parts = market_symbol.split('/')
-    return (len(symbol_parts) == 2 and
-            (symbol_parts[0] == base_currency if base_currency else len(symbol_parts[0]) > 0) and
-            (symbol_parts[1] == quote_currency if quote_currency else len(symbol_parts[1]) > 0))
-
-
 def market_is_active(market: Dict) -> bool:
     """
     Return True if the market is active.

freqtrade/exchange/ftx.py

@@ -1,14 +1,14 @@
 """ FTX exchange subclass """
 import logging
-from typing import Dict
+from typing import Any, Dict
 
 import ccxt
 
-from freqtrade.exceptions import (DDosProtection, ExchangeError,
+from freqtrade.exceptions import (DDosProtection, InsufficientFundsError,
                                   InvalidOrderException, OperationalException,
                                   TemporaryError)
 from freqtrade.exchange import Exchange
-from freqtrade.exchange.common import retrier
+from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier
 
 logger = logging.getLogger(__name__)
 
@@ -20,6 +20,16 @@ class Ftx(Exchange):
         "ohlcv_candle_limit": 1500,
     }
 
+    def market_is_tradable(self, market: Dict[str, Any]) -> bool:
+        """
+        Check if the market symbol is tradable by Freqtrade.
+        Default checks + check if pair is spot pair (no futures trading yet).
+        """
+        parent_check = super().market_is_tradable(market)
+
+        return (parent_check and
+                market.get('spot', False) is True)
+
     def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
         """
         Verify stop_loss against stoploss-order value (limit or price)
@@ -61,7 +71,7 @@ class Ftx(Exchange):
                         'stop price: %s.', pair, stop_price)
             return order
         except ccxt.InsufficientFunds as e:
-            raise ExchangeError(
+            raise InsufficientFundsError(
                 f'Insufficient funds to create {ordertype} sell order on market {pair}. '
                 f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
                 f'Message: {e}') from e
@@ -78,7 +88,7 @@ class Ftx(Exchange):
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    @retrier
+    @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
     def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
             try:

freqtrade/exchange/kraken.py

@@ -1,10 +1,10 @@
 """ Kraken exchange subclass """
 import logging
-from typing import Dict
+from typing import Any, Dict
 
 import ccxt
 
-from freqtrade.exceptions import (DDosProtection, ExchangeError,
+from freqtrade.exceptions import (DDosProtection, InsufficientFundsError,
                                   InvalidOrderException, OperationalException,
                                   TemporaryError)
 from freqtrade.exchange import Exchange
@@ -22,6 +22,16 @@ class Kraken(Exchange):
         "trades_pagination_arg": "since",
     }
 
+    def market_is_tradable(self, market: Dict[str, Any]) -> bool:
+        """
+        Check if the market symbol is tradable by Freqtrade.
+        Default checks + check if pair is darkpool pair.
+        """
+        parent_check = super().market_is_tradable(market)
+
+        return (parent_check and
+                market.get('darkpool', False) is False)
+
     @retrier
     def get_balances(self) -> dict:
         if self._config['dry_run']:
@@ -88,7 +98,7 @@ class Kraken(Exchange):
                         'stop price: %s.', pair, stop_price)
             return order
         except ccxt.InsufficientFunds as e:
-            raise ExchangeError(
+            raise InsufficientFundsError(
                 f'Insufficient funds to create {ordertype} sell order on market {pair}. '
                 f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
                 f'Message: {e}') from e

freqtrade/freqtradebot.py

@@ -17,12 +17,12 @@ from freqtrade.configuration import validate_config_consistency
 from freqtrade.data.converter import order_book_to_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
-from freqtrade.exceptions import (DependencyException, ExchangeError,
+from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, PricingError)
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
-from freqtrade.misc import safe_value_fallback
+from freqtrade.misc import safe_value_fallback, safe_value_fallback2
 from freqtrade.pairlist.pairlistmanager import PairListManager
-from freqtrade.persistence import Trade
+from freqtrade.persistence import Order, Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.rpc import RPCManager, RPCMessageType
 from freqtrade.state import State
@@ -134,6 +134,10 @@ class FreqtradeBot:
             # Adjust stoploss if it was changed
             Trade.stoploss_reinitialization(self.strategy.stoploss)
 
+        # Only update open orders on startup
+        # This will update the database after the initial migration
+        self.update_open_orders()
+
     def process(self) -> None:
         """
         Queries the persistence layer for open trades and handles them,
@@ -144,6 +148,8 @@ class FreqtradeBot:
         # Check whether markets have to be reloaded and reload them when it's needed
         self.exchange.reload_markets()
 
+        self.update_closed_trades_without_assigned_fees()
+
         # Query trades from persistence layer
         trades = Trade.get_open_trades()
 
@@ -227,6 +233,104 @@ class FreqtradeBot:
         open_trades = len(Trade.get_open_trades())
         return max(0, self.config['max_open_trades'] - open_trades)
 
+    def update_open_orders(self):
+        """
+        Updates open orders based on order list kept in the database.
+        Mainly updates the state of orders - but may also close trades
+        """
+        orders = Order.get_open_orders()
+        logger.info(f"Updating {len(orders)} open orders.")
+        for order in orders:
+            try:
+                fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
+                                                                 order.ft_order_side == 'stoploss')
+
+                self.update_trade_state(order.trade, order.order_id, fo)
+
+            except ExchangeError as e:
+                logger.warning(f"Error updating Order {order.order_id} due to {e}")
+
+    def update_closed_trades_without_assigned_fees(self):
+        """
+        Update closed trades without close fees assigned.
+        Only acts when Orders are in the database, otherwise the last orderid is unknown.
+        """
+        trades: List[Trade] = Trade.get_sold_trades_without_assigned_fees()
+        for trade in trades:
+
+            if not trade.is_open and not trade.fee_updated('sell'):
+                # Get sell fee
+                order = trade.select_order('sell', False)
+                if order:
+                    logger.info(f"Updating sell-fee on trade {trade} for order {order.order_id}.")
+                    self.update_trade_state(trade, order.order_id,
+                                            stoploss_order=order.ft_order_side == 'stoploss')
+
+        trades: List[Trade] = Trade.get_open_trades_without_assigned_fees()
+        for trade in trades:
+            if trade.is_open and not trade.fee_updated('buy'):
+                order = trade.select_order('buy', False)
+                if order:
+                    logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.")
+                    self.update_trade_state(trade, order.order_id)
+
+    def handle_insufficient_funds(self, trade: Trade):
+        """
+        Determine if we ever opened a sell order for this trade.
+        If not, try update buy fees - otherwise "refind" the open order we obviously lost.
+        """
+        sell_order = trade.select_order('sell', None)
+        if sell_order:
+            self.refind_lost_order(trade)
+        else:
+            self.reupdate_buy_order_fees(trade)
+
+    def reupdate_buy_order_fees(self, trade: Trade):
+        """
+        Get buy order from database, and try to reupdate.
+        Handles trades where the initial fee-update did not work.
+        """
+        logger.info(f"Trying to reupdate buy fees for {trade}")
+        order = trade.select_order('buy', False)
+        if order:
+            logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.")
+            self.update_trade_state(trade, order.order_id)
+
+    def refind_lost_order(self, trade):
+        """
+        Try refinding a lost trade.
+        Only used when InsufficientFunds appears on sell orders (stoploss or sell).
+        Tries to walk the stored orders and sell them off eventually.
+        """
+        logger.info(f"Trying to refind lost order for {trade}")
+        for order in trade.orders:
+            logger.info(f"Trying to refind {order}")
+            fo = None
+            if not order.ft_is_open:
+                logger.debug(f"Order {order} is no longer open.")
+                continue
+            if order.ft_order_side == 'buy':
+                # Skip buy side - this is handled by reupdate_buy_order_fees
+                continue
+            try:
+                fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
+                                                                 order.ft_order_side == 'stoploss')
+                if order.ft_order_side == 'stoploss':
+                    if fo and fo['status'] == 'open':
+                        # Assume this as the open stoploss order
+                        trade.stoploss_order_id = order.order_id
+                elif order.ft_order_side == 'sell':
+                    if fo and fo['status'] == 'open':
+                        # Assume this as the open order
+                        trade.open_order_id = order.order_id
+                if fo:
+                    logger.info(f"Found {order} for trade {trade}.jj")
+                    self.update_trade_state(trade, order.order_id, fo,
+                                            stoploss_order=order.ft_order_side == 'stoploss')
+
+            except ExchangeError:
+                logger.warning(f"Error updating {order.order_id}.")
+
 #
 # BUY / enter positions / open trades logic and methods
 #
@@ -275,7 +379,7 @@ class FreqtradeBot:
             rate = self._buy_rate_cache.get(pair)
             # Check if cache has been invalidated
             if rate:
-                logger.info(f"Using cached buy rate for {pair}.")
+                logger.debug(f"Using cached buy rate for {pair}.")
                 return rate
 
         bid_strategy = self.config.get('bid_strategy', {})
@@ -433,7 +537,9 @@ class FreqtradeBot:
         """
         logger.debug(f"create_trade for pair {pair}")
 
-        if self.strategy.is_pair_locked(pair):
+        analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe)
+        if self.strategy.is_pair_locked(
+                pair, analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None):
             logger.info(f"Pair {pair} is currently locked.")
             return False
 
@@ -444,7 +550,6 @@ class FreqtradeBot:
             return False
 
         # running get_signal on historical data fetched
-        analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe)
         (buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
 
         if buy and not sell:
@@ -523,15 +628,17 @@ class FreqtradeBot:
                 time_in_force=time_in_force):
             logger.info(f"User requested abortion of buying {pair}")
             return False
-
+        amount = self.exchange.amount_to_precision(pair, amount)
         order = self.exchange.buy(pair=pair, ordertype=order_type,
                                   amount=amount, rate=buy_limit_requested,
                                   time_in_force=time_in_force)
+        order_obj = Order.parse_from_ccxt_object(order, pair, 'buy')
         order_id = order['id']
         order_status = order.get('status', None)
 
         # we assume the order is executed at the price requested
         buy_limit_filled_price = buy_limit_requested
+        amount_requested = amount
 
         if order_status == 'expired' or order_status == 'rejected':
             order_tif = self.strategy.order_time_in_force['buy']
@@ -552,15 +659,14 @@ class FreqtradeBot:
                                order['filled'], order['amount'], order['remaining']
                                )
                 stake_amount = order['cost']
-                amount = order['amount']
-                buy_limit_filled_price = order['price']
-                order_id = None
+                amount = safe_value_fallback(order, 'filled', 'amount')
+                buy_limit_filled_price = safe_value_fallback(order, 'average', 'price')
 
         # in case of FOK the order may be filled immediately and fully
         elif order_status == 'closed':
             stake_amount = order['cost']
-            amount = order['amount']
-            buy_limit_filled_price = order['price']
+            amount = safe_value_fallback(order, 'filled', 'amount')
+            buy_limit_filled_price = safe_value_fallback(order, 'average', 'price')
 
         # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
         fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker')
@@ -568,6 +674,7 @@ class FreqtradeBot:
             pair=pair,
             stake_amount=stake_amount,
             amount=amount,
+            amount_requested=amount_requested,
             fee_open=fee,
             fee_close=fee,
             open_rate=buy_limit_filled_price,
@@ -578,10 +685,11 @@ class FreqtradeBot:
             strategy=self.strategy.get_strategy_name(),
             timeframe=timeframe_to_minutes(self.config['timeframe'])
         )
+        trade.orders.append(order_obj)
 
         # Update fees if order is closed
         if order_status == 'closed':
-            self.update_trade_state(trade, order)
+            self.update_trade_state(trade, order_id, order)
 
         Trade.session.add(trade)
         Trade.session.flush()
@@ -615,7 +723,7 @@ class FreqtradeBot:
         # Send the message
         self.rpc.send_msg(msg)
 
-    def _notify_buy_cancel(self, trade: Trade, order_type: str) -> None:
+    def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
         """
         Sends rpc notification when a buy cancel occured.
         """
@@ -634,6 +742,7 @@ class FreqtradeBot:
             'amount': trade.amount,
             'open_date': trade.open_date,
             'current_rate': current_rate,
+            'reason': reason,
         }
 
         # Send the message
@@ -660,7 +769,7 @@ class FreqtradeBot:
                     trades_closed += 1
 
             except DependencyException as exception:
-                logger.warning('Unable to sell trade: %s', exception)
+                logger.warning('Unable to sell trade %s: %s', trade.pair, exception)
 
         # Updating wallets if any trade occured
         if trades_closed:
@@ -691,7 +800,7 @@ class FreqtradeBot:
             rate = self._sell_rate_cache.get(pair)
             # Check if cache has been invalidated
             if rate:
-                logger.info(f"Using cached sell rate for {pair}.")
+                logger.debug(f"Using cached sell rate for {pair}.")
                 return rate
 
         ask_strategy = self.config.get('ask_strategy', {})
@@ -768,7 +877,7 @@ class FreqtradeBot:
         logger.debug('Found no sell signal for %s.', trade)
         return False
 
-    def create_stoploss_order(self, trade: Trade, stop_price: float, rate: float) -> bool:
+    def create_stoploss_order(self, trade: Trade, stop_price: float) -> bool:
         """
         Abstracts creating stoploss orders from the logic.
         Handles errors and updates the trade database object.
@@ -779,8 +888,16 @@ class FreqtradeBot:
             stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount,
                                                     stop_price=stop_price,
                                                     order_types=self.strategy.order_types)
+
+            order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss')
+            trade.orders.append(order_obj)
             trade.stoploss_order_id = str(stoploss_order['id'])
             return True
+        except InsufficientFundsError as e:
+            logger.warning(f"Unable to place stoploss order {e}.")
+            # Try to figure out what went wrong
+            self.handle_insufficient_funds(trade)
+
         except InvalidOrderException as e:
             trade.stoploss_order_id = None
             logger.error(f'Unable to place a stoploss order on exchange. {e}')
@@ -810,10 +927,14 @@ class FreqtradeBot:
         except InvalidOrderException as exception:
             logger.warning('Unable to fetch stoploss order: %s', exception)
 
+        if stoploss_order:
+            trade.update_order(stoploss_order)
+
         # We check if stoploss order is fulfilled
         if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
             trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value
-            self.update_trade_state(trade, stoploss_order, sl_order=True)
+            self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
+                                    stoploss_order=True)
             # Lock pair for one candle to prevent immediate rebuys
             self.strategy.lock_pair(trade.pair,
                                     timeframe_to_next_date(self.config['timeframe']))
@@ -831,14 +952,13 @@ class FreqtradeBot:
             stoploss = self.edge.stoploss(pair=trade.pair) if self.edge else self.strategy.stoploss
             stop_price = trade.open_rate * (1 + stoploss)
 
-            if self.create_stoploss_order(trade=trade, stop_price=stop_price, rate=stop_price):
-                trade.stoploss_last_update = datetime.now()
+            if self.create_stoploss_order(trade=trade, stop_price=stop_price):
+                trade.stoploss_last_update = datetime.utcnow()
                 return False
 
         # If stoploss order is canceled for some reason we add it
         if stoploss_order and stoploss_order['status'] in ('canceled', 'cancelled'):
-            if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss,
-                                          rate=trade.stop_loss):
+            if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss):
                 return False
             else:
                 trade.stoploss_order_id = None
@@ -866,17 +986,17 @@ class FreqtradeBot:
             update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60)
             if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat:
                 # cancelling the current stoploss on exchange first
-                logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s}) '
-                            'in order to add another one ...', order['id'])
+                logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} "
+                            f"(orderid:{order['id']}) in order to add another one ...")
                 try:
-                    self.exchange.cancel_stoploss_order(order['id'], trade.pair)
+                    co = self.exchange.cancel_stoploss_order(order['id'], trade.pair)
+                    trade.update_order(co)
                 except InvalidOrderException:
                     logger.exception(f"Could not cancel stoploss order {order['id']} "
                                      f"for pair {trade.pair}")
 
                 # Create new stoploss order
-                if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss,
-                                                  rate=trade.stop_loss):
+                if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss):
                     logger.warning(f"Could not create trailing stoploss order "
                                    f"for pair {trade.pair}.")
 
@@ -921,11 +1041,11 @@ class FreqtradeBot:
                 if not trade.open_order_id:
                     continue
                 order = self.exchange.fetch_order(trade.open_order_id, trade.pair)
-            except (ExchangeError, InvalidOrderException):
+            except (ExchangeError):
                 logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
                 continue
 
-            fully_cancelled = self.update_trade_state(trade, order)
+            fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order)
 
             if (order['side'] == 'buy' and (order['status'] == 'open' or fully_cancelled) and (
                     fully_cancelled
@@ -954,7 +1074,7 @@ class FreqtradeBot:
         for trade in Trade.get_open_order_trades():
             try:
                 order = self.exchange.fetch_order(trade.open_order_id, trade.pair)
-            except (DependencyException, InvalidOrderException):
+            except (ExchangeError):
                 logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
                 continue
 
@@ -973,9 +1093,14 @@ class FreqtradeBot:
 
         # Cancelled orders may have the status of 'canceled' or 'closed'
         if order['status'] not in ('canceled', 'closed'):
-            reason = constants.CANCEL_REASON['TIMEOUT']
             corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
                                                             trade.amount)
+            # Avoid race condition where the order could not be cancelled coz its already filled.
+            # Simply bailing here is the only safe way - as this order will then be
+            # handled in the next iteration.
+            if corder.get('status') not in ('canceled', 'closed'):
+                logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.")
+                return False
         else:
             # Order was cancelled already, so we can reuse the existing dict
             corder = order
@@ -984,14 +1109,13 @@ class FreqtradeBot:
         logger.info('Buy order %s for %s.', reason, trade)
 
         # Using filled to determine the filled amount
-        filled_amount = safe_value_fallback(corder, order, 'filled', 'filled')
-
+        filled_amount = safe_value_fallback2(corder, order, 'filled', 'filled')
         if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
             logger.info('Buy order fully cancelled. Removing %s from database.', trade)
             # if trade is not partially completed, just delete the trade
-            Trade.session.delete(trade)
-            Trade.session.flush()
+            trade.delete()
             was_trade_fully_canceled = True
+            reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
         else:
             # if trade is partially complete, edit the stake details for the trade
             # and close the order
@@ -1000,17 +1124,15 @@ class FreqtradeBot:
             # we need to fall back to the values from order if corder does not contain these keys.
             trade.amount = filled_amount
             trade.stake_amount = trade.amount * trade.open_rate
-            self.update_trade_state(trade, corder, trade.amount)
+            self.update_trade_state(trade, trade.open_order_id, corder)
 
             trade.open_order_id = None
             logger.info('Partial buy order timeout for %s.', trade)
-            self.rpc.send_msg({
-                'type': RPCMessageType.STATUS_NOTIFICATION,
-                'status': f'Remaining buy order for {trade.pair} cancelled due to timeout'
-            })
+            reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}"
 
         self.wallets.update()
-        self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'])
+        self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'],
+                                reason=reason)
         return was_trade_fully_canceled
 
     def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str:
@@ -1041,7 +1163,7 @@ class FreqtradeBot:
             trade.open_order_id = None
         else:
             # TODO: figure out how to handle partially complete sell orders
-            reason = constants.CANCEL_REASON['PARTIALLY_FILLED']
+            reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
 
         self.wallets.update()
         self._notify_sell_cancel(
@@ -1116,19 +1238,28 @@ class FreqtradeBot:
             logger.info(f"User requested abortion of selling {trade.pair}")
             return False
 
+        try:
             # Execute sell and update trade record
-        order = self.exchange.sell(pair=str(trade.pair),
+            order = self.exchange.sell(pair=trade.pair,
                                        ordertype=order_type,
                                        amount=amount, rate=limit,
                                        time_in_force=time_in_force
                                        )
+        except InsufficientFundsError as e:
+            logger.warning(f"Unable to place order {e}.")
+            # Try to figure out what went wrong
+            self.handle_insufficient_funds(trade)
+            return False
+
+        order_obj = Order.parse_from_ccxt_object(order, trade.pair, 'sell')
+        trade.orders.append(order_obj)
 
         trade.open_order_id = order['id']
         trade.close_rate_requested = limit
         trade.sell_reason = sell_reason.value
         # In case of market sell orders the order can be closed immediately
         if order.get('status', 'unknown') == 'closed':
-            self.update_trade_state(trade, order)
+            self.update_trade_state(trade, trade.open_order_id, order)
         Trade.session.flush()
 
         # Lock pair for one candle to prevent immediate rebuys
@@ -1225,31 +1356,37 @@ class FreqtradeBot:
 # Common update trade state methods
 #
 
-    def update_trade_state(self, trade: Trade, action_order: dict = None,
-                           order_amount: float = None, sl_order: bool = False) -> bool:
+    def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None,
+                           stoploss_order: bool = False) -> bool:
         """
         Checks trades with open orders and updates the amount if necessary
         Handles closing both buy and sell orders.
+        :param trade: Trade object of the trade we're analyzing
+        :param order_id: Order-id of the order we're analyzing
+        :param action_order: Already aquired order object
         :return: True if order has been cancelled without being filled partially, False otherwise
         """
-        # Get order details for actual price per unit
-        if trade.open_order_id:
-            order_id = trade.open_order_id
-        elif trade.stoploss_order_id and sl_order:
-            order_id = trade.stoploss_order_id
-        else:
+        if not order_id:
+            logger.warning(f'Orderid for trade {trade} is empty.')
             return False
+
         # Update trade with order values
         logger.info('Found open order for %s', trade)
         try:
-            order = action_order or self.exchange.fetch_order(order_id, trade.pair)
+            order = action_order or self.exchange.fetch_order_or_stoploss_order(order_id,
+                                                                                trade.pair,
+                                                                                stoploss_order)
         except InvalidOrderException as exception:
             logger.warning('Unable to fetch order %s: %s', order_id, exception)
             return False
+
+        trade.update_order(order)
+
         # Try update amount (binance-fix)
         try:
-            new_amount = self.get_real_amount(trade, order, order_amount)
-            if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
+            new_amount = self.get_real_amount(trade, order)
+            if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount,
+                           abs_tol=constants.MATH_CLOSE_PREC):
                 order['amount'] = new_amount
                 order.pop('filled', None)
                 trade.recalc_open_trade_price()
@@ -1285,7 +1422,7 @@ class FreqtradeBot:
             return real_amount
         return amount
 
-    def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float:
+    def get_real_amount(self, trade: Trade, order: Dict) -> float:
         """
         Detect and update trade fee.
         Calls trade.update_fee() uppon correct detection.
@@ -1294,8 +1431,7 @@ class FreqtradeBot:
         :return: identical (or new) amount for the trade
         """
         # Init variables
-        if order_amount is None:
-            order_amount = order['amount']
+        order_amount = safe_value_fallback(order, 'filled', 'amount')
         # Only run for closed orders
         if trade.fee_updated(order.get('side', '')) or order['status'] == 'open':
             return order_amount
@@ -1319,7 +1455,7 @@ class FreqtradeBot:
         """
         fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee.
         """
-        trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair,
+        trades = self.exchange.get_trades_for_order(order['id'], trade.pair,
                                                     trade.open_date)
 
         if len(trades) == 0:

freqtrade/loggers.py

@@ -1,14 +1,18 @@
 import logging
 import sys
-
 from logging import Formatter
-from logging.handlers import RotatingFileHandler, SysLogHandler
-from typing import Any, Dict, List
+from logging.handlers import (BufferingHandler, RotatingFileHandler,
+                              SysLogHandler)
+from typing import Any, Dict
 
 from freqtrade.exceptions import OperationalException
 
-
 logger = logging.getLogger(__name__)
+LOGFORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+
+# Initialize bufferhandler - will be used for /log endpoints
+bufferHandler = BufferingHandler(1000)
+bufferHandler.setFormatter(Formatter(LOGFORMAT))
 
 
 def _set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None:
@@ -33,17 +37,31 @@ def _set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None:
     )
 
 
+def setup_logging_pre() -> None:
+    """
+    Early setup for logging.
+    Uses INFO loglevel and only the Streamhandler.
+    Early messages (before proper logging setup) will therefore only be sent to additional
+    logging handlers after the real initialization, because we don't know which
+    ones the user desires beforehand.
+    """
+    logging.basicConfig(
+        level=logging.INFO,
+        format=LOGFORMAT,
+        handlers=[logging.StreamHandler(sys.stderr), bufferHandler]
+    )
+
+
 def setup_logging(config: Dict[str, Any]) -> None:
     """
     Process -v/--verbose, --logfile options
     """
     # Log level
     verbosity = config['verbosity']
-
-    # Log to stderr
-    log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)]
+    logging.root.addHandler(bufferHandler)
 
     logfile = config.get('logfile')
+
     if logfile:
         s = logfile.split(':')
         if s[0] == 'syslog':
@@ -58,28 +76,27 @@ def setup_logging(config: Dict[str, Any]) -> None:
             # to perform reduction of repeating messages if this is set in the
             # syslog config. The messages should be equal for this.
             handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
-            log_handlers.append(handler)
+            logging.root.addHandler(handler)
         elif s[0] == 'journald':
             try:
                 from systemd.journal import JournaldLogHandler
             except ImportError:
                 raise OperationalException("You need the systemd python package be installed in "
                                            "order to use logging to journald.")
-            handler = JournaldLogHandler()
+            handler_jd = JournaldLogHandler()
             # No datetime field for logging into journald, to allow syslog
             # to perform reduction of repeating messages if this is set in the
             # syslog config. The messages should be equal for this.
-            handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
-            log_handlers.append(handler)
+            handler_jd.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
+            logging.root.addHandler(handler_jd)
         else:
-            log_handlers.append(RotatingFileHandler(logfile,
-                                                    maxBytes=1024 * 1024,  # 1Mb
-                                                    backupCount=10))
+            handler_rf = RotatingFileHandler(logfile,
+                                             maxBytes=1024 * 1024 * 10,  # 10Mb
+                                             backupCount=10)
+            handler_rf.setFormatter(Formatter(LOGFORMAT))
+            logging.root.addHandler(handler_rf)
 
-    logging.basicConfig(
-        level=logging.INFO if verbosity < 1 else logging.DEBUG,
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        handlers=log_handlers
-    )
+    logging.root.setLevel(logging.INFO if verbosity < 1 else logging.DEBUG)
     _set_loggers(verbosity, config.get('api_server', {}).get('verbosity', 'info'))
+
     logger.info('Verbosity set to %s', verbosity)

freqtrade/main.py

@@ -3,18 +3,17 @@
 Main Freqtrade bot script.
 Read the documentation to know what cli arguments you need.
 """
-
-from freqtrade.exceptions import FreqtradeException, OperationalException
+import logging
 import sys
+from typing import Any, List
+
 # check min. python version
 if sys.version_info < (3, 6):
     sys.exit("Freqtrade requires Python version >= 3.6")
 
-# flake8: noqa E402
-import logging
-from typing import Any, List
-
 from freqtrade.commands import Arguments
+from freqtrade.exceptions import FreqtradeException, OperationalException
+from freqtrade.loggers import setup_logging_pre
 
 
 logger = logging.getLogger('freqtrade')
@@ -28,6 +27,7 @@ def main(sysargv: List[str] = None) -> None:
 
     return_code: Any = 1
     try:
+        setup_logging_pre()
         arguments = Arguments(sysargv)
         args = arguments.get_parsed_arg()
 

freqtrade/misc.py

@@ -134,7 +134,21 @@ def round_dict(d, n):
     return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()}
 
 
-def safe_value_fallback(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None):
+def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None):
+    """
+    Search a value in obj, return this if it's not None.
+    Then search key2 in obj - return that if it's not none - then use default_value.
+    Else falls back to None.
+    """
+    if key1 in obj and obj[key1] is not None:
+        return obj[key1]
+    else:
+        if key2 in obj and obj[key2] is not None:
+            return obj[key2]
+    return default_value
+
+
+def safe_value_fallback2(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None):
     """
     Search a value in dict1, return this if it's not None.
     Fall back to dict2 - return key2 from dict2 if it's not None.

freqtrade/optimize/backtesting.py

@@ -13,6 +13,7 @@ from pandas import DataFrame
 
 from freqtrade.configuration import (TimeRange, remove_credentials,
                                      validate_config_consistency)
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
 from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.dataprovider import DataProvider
@@ -20,11 +21,10 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats,
                                                  show_backtest_results,
-                                                 store_backtest_result)
+                                                 store_backtest_stats)
 from freqtrade.pairlist.pairlistmanager import PairListManager
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.state import RunMode
 from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
 
 logger = logging.getLogger(__name__)
@@ -37,14 +37,15 @@ class BacktestResult(NamedTuple):
     pair: str
     profit_percent: float
     profit_abs: float
-    open_time: datetime
-    close_time: datetime
-    open_index: int
-    close_index: int
+    open_date: datetime
+    open_rate: float
+    open_fee: float
+    close_date: datetime
+    close_rate: float
+    close_fee: float
+    amount: float
     trade_duration: float
     open_at_end: bool
-    open_rate: float
-    close_rate: float
     sell_reason: SellType
 
 
@@ -65,9 +66,8 @@ class Backtesting:
         self.strategylist: List[IStrategy] = []
         self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
 
-        if self.config.get('runmode') != RunMode.HYPEROPT:
-            self.dataprovider = DataProvider(self.config, self.exchange)
-            IStrategy.dp = self.dataprovider
+        dataprovider = DataProvider(self.config, self.exchange)
+        IStrategy.dp = dataprovider
 
         if self.config.get('strategy_list', None):
             for strat in list(self.config['strategy_list']):
@@ -96,6 +96,7 @@ class Backtesting:
                 "PrecisionFilter not allowed for backtesting multiple strategies."
             )
 
+        dataprovider.add_pairlisthandler(self.pairlists)
         self.pairlists.refresh_pairlist()
 
         if len(self.pairlists.whitelist) == 0:
@@ -137,10 +138,10 @@ class Backtesting:
 
         min_date, max_date = history.get_timerange(data)
 
-        logger.info(
-            'Loading data from %s up to %s (%s days)..',
-            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
-        )
+        logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'({(max_date - min_date).days} days)..')
+
         # Adjust startts forward if not enough data is available
         timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
                                             self.required_startup, min_date)
@@ -225,7 +226,7 @@ class Backtesting:
             open_rate=buy_row.open,
             open_date=buy_row.date,
             stake_amount=stake_amount,
-            amount=stake_amount / buy_row.open,
+            amount=round(stake_amount / buy_row.open, 8),
             fee_open=self.fee,
             fee_close=self.fee,
             is_open=True,
@@ -246,14 +247,15 @@ class Backtesting:
                 return BacktestResult(pair=pair,
                                       profit_percent=trade.calc_profit_ratio(rate=closerate),
                                       profit_abs=trade.calc_profit(rate=closerate),
-                                      open_time=buy_row.date,
-                                      close_time=sell_row.date,
-                                      trade_duration=trade_dur,
-                                      open_index=buy_row.Index,
-                                      close_index=sell_row.Index,
-                                      open_at_end=False,
+                                      open_date=buy_row.date,
                                       open_rate=buy_row.open,
+                                      open_fee=self.fee,
+                                      close_date=sell_row.date,
                                       close_rate=closerate,
+                                      close_fee=self.fee,
+                                      amount=trade.amount,
+                                      trade_duration=trade_dur,
+                                      open_at_end=False,
                                       sell_reason=sell.sell_type
                                       )
         if partial_ohlcv:
@@ -262,15 +264,16 @@ class Backtesting:
             bt_res = BacktestResult(pair=pair,
                                     profit_percent=trade.calc_profit_ratio(rate=sell_row.open),
                                     profit_abs=trade.calc_profit(rate=sell_row.open),
-                                    open_time=buy_row.date,
-                                    close_time=sell_row.date,
+                                    open_date=buy_row.date,
+                                    open_rate=buy_row.open,
+                                    open_fee=self.fee,
+                                    close_date=sell_row.date,
+                                    close_rate=sell_row.open,
+                                    close_fee=self.fee,
+                                    amount=trade.amount,
                                     trade_duration=int((
                                         sell_row.date - buy_row.date).total_seconds() // 60),
-                                    open_index=buy_row.Index,
-                                    close_index=sell_row.Index,
                                     open_at_end=True,
-                                    open_rate=buy_row.open,
-                                    close_rate=sell_row.open,
                                     sell_reason=SellType.FORCE_SELL
                                     )
             logger.debug(f"{pair} - Force selling still open trade, "
@@ -356,8 +359,8 @@ class Backtesting:
 
                 if trade_entry:
                     logger.debug(f"{pair} - Locking pair till "
-                                 f"close_time={trade_entry.close_time}")
-                    lock_pair_until[pair] = trade_entry.close_time
+                                 f"close_date={trade_entry.close_date}")
+                    lock_pair_until[pair] = trade_entry.close_date
                     trades.append(trade_entry)
                 else:
                     # Set lock_pair_until to end of testing period if trade could not be closed
@@ -377,12 +380,6 @@ class Backtesting:
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
 
-        # Use max_open_trades in backtesting, except --disable-max-market-positions is set
-        if self.config.get('use_max_market_positions', True):
-            max_open_trades = self.config['max_open_trades']
-        else:
-            logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
-            max_open_trades = 0
         position_stacking = self.config.get('position_stacking', False)
 
         data, timerange = self.load_bt_data()
@@ -392,6 +389,15 @@ class Backtesting:
             logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
             self._set_strategy(strat)
 
+            # Use max_open_trades in backtesting, except --disable-max-market-positions is set
+            if self.config.get('use_max_market_positions', True):
+                # Must come from strategy config, as the strategy may modify this setting.
+                max_open_trades = self.strategy.config['max_open_trades']
+            else:
+                logger.info(
+                    'Ignoring max_open_trades (--disable-max-market-positions was used) ...')
+                max_open_trades = 0
+
             # need to reprocess data every time to populate signals
             preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
 
@@ -400,12 +406,11 @@ class Backtesting:
                 preprocessed[pair] = trim_dataframe(df, timerange)
             min_date, max_date = history.get_timerange(preprocessed)
 
-            logger.info(
-                'Backtesting with data from %s up to %s (%s days)..',
-                min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
-            )
+            logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
+                        f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
+                        f'({(max_date - min_date).days} days)..')
             # Execute backtest and print results
-            all_results[self.strategy.get_strategy_name()] = self.backtest(
+            results = self.backtest(
                 processed=preprocessed,
                 stake_amount=self.config['stake_amount'],
                 start_date=min_date,
@@ -413,9 +418,15 @@ class Backtesting:
                 max_open_trades=max_open_trades,
                 position_stacking=position_stacking,
             )
+            all_results[self.strategy.get_strategy_name()] = {
+                'results': results,
+                'config': self.strategy.config,
+            }
+
+        stats = generate_backtest_stats(data, all_results, min_date=min_date, max_date=max_date)
 
         if self.config.get('export', False):
-            store_backtest_result(self.config['exportfilename'], all_results)
+            store_backtest_stats(self.config['exportfilename'], stats)
+
         # Show backtest results
-        stats = generate_backtest_stats(self.config, data, all_results)
         show_backtest_results(self.config, stats)

freqtrade/optimize/hyperopt.py

@@ -4,27 +4,28 @@
 This module contains the hyperopt logic
 """
 
+import io
 import locale
 import logging
 import random
 import warnings
-from math import ceil
 from collections import OrderedDict
+from math import ceil
 from operator import itemgetter
 from pathlib import Path
 from pprint import pformat
 from typing import Any, Dict, List, Optional
 
+import progressbar
 import rapidjson
+import tabulate
 from colorama import Fore, Style
+from colorama import init as colorama_init
 from joblib import (Parallel, cpu_count, delayed, dump, load,
                     wrap_non_picklable_objects)
-from pandas import DataFrame, json_normalize, isna
-import progressbar
-import tabulate
-from os import path
-import io
+from pandas import DataFrame, isna, json_normalize
 
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.history import get_timerange
 from freqtrade.exceptions import OperationalException
@@ -32,9 +33,11 @@ from freqtrade.misc import plural, round_dict
 from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
 from freqtrade.optimize.hyperopt_interface import IHyperOpt  # noqa: F401
-from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss  # noqa: F401
+from freqtrade.optimize.hyperopt_loss_interface import \
+    IHyperOptLoss  # noqa: F401
 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver,
                                                    HyperOptResolver)
+from freqtrade.strategy import IStrategy
 
 # Suppress scikit-learn FutureWarnings from skopt
 with warnings.catch_warnings():
@@ -312,12 +315,18 @@ class Hyperopt:
 
         trials = json_normalize(results, max_level=1)
         trials['Best'] = ''
+        if 'results_metrics.winsdrawslosses' not in trials.columns:
+            # Ensure compatibility with older versions of hyperopt results
+            trials['results_metrics.winsdrawslosses'] = 'N/A'
+
         trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
+                         'results_metrics.winsdrawslosses',
                          'results_metrics.avg_profit', 'results_metrics.total_profit',
                          'results_metrics.profit', 'results_metrics.duration',
                          'loss', 'is_initial_point', 'is_best']]
-        trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit',
-                          'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
+        trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
+                          'Total profit', 'Profit', 'Avg duration', 'Objective',
+                          'is_initial_point', 'is_best']
         trials['is_profit'] = False
         trials.loc[trials['is_initial_point'], 'Best'] = '*     '
         trials.loc[trials['is_best'], 'Best'] = 'Best'
@@ -390,7 +399,7 @@ class Hyperopt:
             return
 
         # Verification for overwrite
-        if path.isfile(csv_file):
+        if Path(csv_file).is_file():
             logger.error(f"CSV file already exists: {csv_file}")
             return
 
@@ -558,9 +567,17 @@ class Hyperopt:
         }
 
     def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict:
+        wins = len(backtesting_results[backtesting_results.profit_percent > 0])
+        draws = len(backtesting_results[backtesting_results.profit_percent == 0])
+        losses = len(backtesting_results[backtesting_results.profit_percent < 0])
         return {
             'trade_count': len(backtesting_results.index),
+            'wins': wins,
+            'draws': draws,
+            'losses': losses,
+            'winsdrawslosses': f"{wins:>4} {draws:>4} {losses:>4}",
             'avg_profit': backtesting_results.profit_percent.mean() * 100.0,
+            'median_profit': backtesting_results.profit_percent.median() * 100.0,
             'total_profit': backtesting_results.profit_abs.sum(),
             'profit': backtesting_results.profit_percent.sum() * 100.0,
             'duration': backtesting_results.trade_duration.mean(),
@@ -572,7 +589,10 @@ class Hyperopt:
         """
         stake_cur = self.config['stake_currency']
         return (f"{results_metrics['trade_count']:6d} trades. "
+                f"{results_metrics['wins']}/{results_metrics['draws']}"
+                f"/{results_metrics['losses']} Wins/Draws/Losses. "
                 f"Avg profit {results_metrics['avg_profit']: 6.2f}%. "
+                f"Median profit {results_metrics['median_profit']: 6.2f}%. "
                 f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} "
                 f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
                 f"Avg duration {results_metrics['duration']:5.1f} min."
@@ -625,15 +645,17 @@ class Hyperopt:
             preprocessed[pair] = trim_dataframe(df, timerange)
         min_date, max_date = get_timerange(data)
 
-        logger.info(
-            'Hyperopting with data from %s up to %s (%s days)..',
-            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
-        )
+        logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'({(max_date - min_date).days} days)..')
+
         dump(preprocessed, self.data_pickle_file)
 
         # We don't need exchange instance anymore while running hyperopt
         self.backtesting.exchange = None  # type: ignore
         self.backtesting.pairlists = None  # type: ignore
+        self.backtesting.strategy.dp = None  # type: ignore
+        IStrategy.dp = None  # type: ignore
 
         self.epochs = self.load_previous_results(self.results_file)
 
@@ -644,6 +666,10 @@ class Hyperopt:
 
         self.dimensions: List[Dimension] = self.hyperopt_space()
         self.opt = self.get_optimizer(self.dimensions, config_jobs)
+
+        if self.print_colorized:
+            colorama_init(autoreset=True)
+
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
                 jobs = parallel._effective_n_jobs()

freqtrade/optimize/hyperopt_loss_sharpe_daily.py

@@ -43,7 +43,7 @@ class SharpeHyperOptLossDaily(IHyperOptLoss):
                              normalize=True)
 
         sum_daily = (
-            results.resample(resample_freq, on='close_time').agg(
+            results.resample(resample_freq, on='close_date').agg(
                 {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0)
         )
 

freqtrade/optimize/hyperopt_loss_sortino_daily.py

@@ -45,7 +45,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss):
                              normalize=True)
 
         sum_daily = (
-            results.resample(resample_freq, on='close_time').agg(
+            results.resample(resample_freq, on='close_date').agg(
                 {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0)
         )
 

freqtrade/optimize/optimize_reports.py

@@ -1,46 +1,40 @@
 import logging
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Union
 
+from arrow import Arrow
 from pandas import DataFrame
+from numpy import int64
 from tabulate import tabulate
 
+from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
+from freqtrade.data.btanalysis import calculate_max_drawdown, calculate_market_change
 from freqtrade.misc import file_dump_json
 
 logger = logging.getLogger(__name__)
 
 
-def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame]) -> None:
+def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> None:
     """
-    Stores backtest results to file (one file per strategy)
-    :param recordfilename: Destination filename
-    :param all_results: Dict of Dataframes, one results dataframe per strategy
+    Stores backtest results
+    :param recordfilename: Path object, which can either be a filename or a directory.
+        Filenames will be appended with a timestamp right before the suffix
+        while for diectories, <directory>/backtest-result-<datetime>.json will be used as filename
+    :param stats: Dataframe containing the backtesting statistics
     """
-    for strategy, results in all_results.items():
-        records = backtest_result_to_list(results)
-
-        if records:
-            filename = recordfilename
-            if len(all_results) > 1:
-                # Inject strategy to filename
+    if recordfilename.is_dir():
+        filename = (recordfilename /
+                    f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.json')
+    else:
         filename = Path.joinpath(
             recordfilename.parent,
-                    f'{recordfilename.stem}-{strategy}').with_suffix(recordfilename.suffix)
-            logger.info(f'Dumping backtest results to {filename}')
-            file_dump_json(filename, records)
+            f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
+            ).with_suffix(recordfilename.suffix)
+    file_dump_json(filename, stats)
 
-
-def backtest_result_to_list(results: DataFrame) -> List[List]:
-    """
-    Converts a list of Backtest-results to list
-    :param results: Dataframe containing results for one strategy
-    :return: List of Lists containing the trades
-    """
-    return [[t.pair, t.profit_percent, t.open_time.timestamp(),
-             t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
-             t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value]
-            for index, t in results.iterrows()]
+    latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN)
+    file_dump_json(latest_filename, {'latest_backtest': str(filename.name)})
 
 
 def _get_line_floatfmt() -> List[str]:
@@ -66,11 +60,12 @@ def _generate_result_line(result: DataFrame, max_open_trades: int, first_column:
     return {
         'key': first_column,
         'trades': len(result),
-        'profit_mean': result['profit_percent'].mean(),
-        'profit_mean_pct': result['profit_percent'].mean() * 100.0,
+        'profit_mean': result['profit_percent'].mean() if len(result) > 0 else 0.0,
+        'profit_mean_pct': result['profit_percent'].mean() * 100.0 if len(result) > 0 else 0.0,
         'profit_sum': result['profit_percent'].sum(),
         'profit_sum_pct': result['profit_percent'].sum() * 100.0,
         'profit_total_abs': result['profit_abs'].sum(),
+        'profit_total': result['profit_percent'].sum() / max_open_trades,
         'profit_total_pct': result['profit_percent'].sum() * 100.0 / max_open_trades,
         'duration_avg': str(timedelta(
                             minutes=round(result['trade_duration'].mean()))
@@ -127,7 +122,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
 
         profit_mean = result['profit_percent'].mean()
         profit_sum = result["profit_percent"].sum()
-        profit_percent_tot = round(result['profit_percent'].sum() * 100.0 / max_open_trades, 2)
+        profit_percent_tot = result['profit_percent'].sum() / max_open_trades
 
         tabular_data.append(
             {
@@ -141,25 +136,25 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
                 'profit_sum': profit_sum,
                 'profit_sum_pct': round(profit_sum * 100, 2),
                 'profit_total_abs': result['profit_abs'].sum(),
-                'profit_pct_total': profit_percent_tot,
+                'profit_total': profit_percent_tot,
+                'profit_total_pct': round(profit_percent_tot * 100, 2),
             }
         )
     return tabular_data
 
 
-def generate_strategy_metrics(stake_currency: str, max_open_trades: int,
-                              all_results: Dict) -> List[Dict]:
+def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
     """
     Generate summary per strategy
-    :param stake_currency: stake-currency - used to correctly name headers
-    :param max_open_trades: Maximum allowed open trades used for backtest
     :param all_results: Dict of <Strategyname: BacktestResult> containing results for all strategies
     :return: List of Dicts containing the metrics per Strategy
     """
 
     tabular_data = []
     for strategy, results in all_results.items():
-        tabular_data.append(_generate_result_line(results, max_open_trades, strategy))
+        tabular_data.append(_generate_result_line(
+            results['results'], results['config']['max_open_trades'], strategy)
+            )
     return tabular_data
 
 
@@ -189,19 +184,63 @@ def generate_edge_table(results: dict) -> str:
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")  # type: ignore
 
 
-def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
-                            all_results: Dict[str, DataFrame]) -> Dict[str, Any]:
+def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
+    if len(results) == 0:
+        return {
+            'backtest_best_day': 0,
+            'backtest_worst_day': 0,
+            'winning_days': 0,
+            'draw_days': 0,
+            'losing_days': 0,
+            'winner_holding_avg': timedelta(),
+            'loser_holding_avg': timedelta(),
+        }
+    daily_profit = results.resample('1d', on='close_date')['profit_percent'].sum()
+    worst = min(daily_profit)
+    best = max(daily_profit)
+    winning_days = sum(daily_profit > 0)
+    draw_days = sum(daily_profit == 0)
+    losing_days = sum(daily_profit < 0)
+
+    winning_trades = results.loc[results['profit_percent'] > 0]
+    losing_trades = results.loc[results['profit_percent'] < 0]
+
+    return {
+        'backtest_best_day': best,
+        'backtest_worst_day': worst,
+        'winning_days': winning_days,
+        'draw_days': draw_days,
+        'losing_days': losing_days,
+        'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
+                               if not winning_trades.empty else timedelta()),
+        'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
+                              if not losing_trades.empty else timedelta()),
+    }
+
+
+def generate_backtest_stats(btdata: Dict[str, DataFrame],
+                            all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
+                            min_date: Arrow, max_date: Arrow
+                            ) -> Dict[str, Any]:
     """
-    :param config: Configuration object used for backtest
     :param btdata: Backtest data
-    :param all_results: backtest result - dictionary with { Strategy: results}.
+    :param all_results: backtest result - dictionary in the form:
+                     { Strategy: {'results: results, 'config: config}}.
+    :param min_date: Backtest start date
+    :param max_date: Backtest end date
     :return:
     Dictionary containing results per strategy and a stratgy summary.
     """
-    stake_currency = config['stake_currency']
-    max_open_trades = config['max_open_trades']
     result: Dict[str, Any] = {'strategy': {}}
-    for strategy, results in all_results.items():
+    market_change = calculate_market_change(btdata, 'close')
+
+    for strategy, content in all_results.items():
+        results: Dict[str, DataFrame] = content['results']
+        if not isinstance(results, DataFrame):
+            continue
+        config = content['config']
+        max_open_trades = config['max_open_trades']
+        stake_currency = config['stake_currency']
 
         pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
                                              max_open_trades=max_open_trades,
@@ -212,17 +251,69 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
                                                   max_open_trades=max_open_trades,
                                                   results=results.loc[results['open_at_end']],
                                                   skip_nan=True)
+        daily_stats = generate_daily_stats(results)
+
+        results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
+        results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
+
+        backtest_days = (max_date - min_date).days
         strat_stats = {
-            'trades': backtest_result_to_list(results),
+            'trades': results.to_dict(orient='records'),
             'results_per_pair': pair_results,
             'sell_reason_summary': sell_reason_stats,
             'left_open_trades': left_open_results,
+            'total_trades': len(results),
+            'profit_mean': results['profit_percent'].mean() if len(results) > 0 else 0,
+            'profit_total': results['profit_percent'].sum(),
+            'profit_total_abs': results['profit_abs'].sum(),
+            'backtest_start': min_date.datetime,
+            'backtest_start_ts': min_date.timestamp * 1000,
+            'backtest_end': max_date.datetime,
+            'backtest_end_ts': max_date.timestamp * 1000,
+            'backtest_days': backtest_days,
+
+            'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
+            'market_change': market_change,
+            'pairlist': list(btdata.keys()),
+            'stake_amount': config['stake_amount'],
+            'stake_currency': config['stake_currency'],
+            'max_open_trades': (config['max_open_trades']
+                                if config['max_open_trades'] != float('inf') else -1),
+            'timeframe': config['timeframe'],
+            # Parameters relevant for backtesting
+            'stoploss': config['stoploss'],
+            'trailing_stop': config.get('trailing_stop', False),
+            'trailing_stop_positive': config.get('trailing_stop_positive'),
+            'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0),
+            'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
+            'minimal_roi': config['minimal_roi'],
+            'use_sell_signal': config['ask_strategy']['use_sell_signal'],
+            'sell_profit_only': config['ask_strategy']['sell_profit_only'],
+            'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
+            **daily_stats,
         }
         result['strategy'][strategy] = strat_stats
 
-    strategy_results = generate_strategy_metrics(stake_currency=stake_currency,
-                                                 max_open_trades=max_open_trades,
-                                                 all_results=all_results)
+        try:
+            max_drawdown, drawdown_start, drawdown_end = calculate_max_drawdown(
+                results, value_col='profit_percent')
+            strat_stats.update({
+                'max_drawdown': max_drawdown,
+                'drawdown_start': drawdown_start,
+                'drawdown_start_ts': drawdown_start.timestamp() * 1000,
+                'drawdown_end': drawdown_end,
+                'drawdown_end_ts': drawdown_end.timestamp() * 1000,
+            })
+        except ValueError:
+            strat_stats.update({
+                'max_drawdown': 0.0,
+                'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc),
+                'drawdown_start_ts': 0,
+                'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc),
+                'drawdown_end_ts': 0,
+            })
+
+    strategy_results = generate_strategy_metrics(all_results=all_results)
 
     result['strategy_comparison'] = strategy_results
 
@@ -273,7 +364,7 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
 
     output = [[
         t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'],
-        t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_pct_total'],
+        t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_total_pct'],
     ] for t in sell_reason_stats]
     return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
 
@@ -298,6 +389,35 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
 
 
+def text_table_add_metrics(strat_results: Dict) -> str:
+    if len(strat_results['trades']) > 0:
+        min_trade = min(strat_results['trades'], key=lambda x: x['open_date'])
+        metrics = [
+            ('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Total trades', strat_results['total_trades']),
+            ('First trade', min_trade['open_date'].strftime(DATETIME_PRINT_FORMAT)),
+            ('First trade Pair', min_trade['pair']),
+            ('Total Profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"),
+            ('Trades per day', strat_results['trades_per_day']),
+            ('Best day', f"{round(strat_results['backtest_best_day'] * 100, 2)}%"),
+            ('Worst day', f"{round(strat_results['backtest_worst_day'] * 100, 2)}%"),
+            ('Days win/draw/lose', f"{strat_results['winning_days']} / "
+                f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
+            ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
+            ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
+            ('', ''),  # Empty line to improve readability
+            ('Max Drawdown', f"{round(strat_results['max_drawdown'] * 100, 2)}%"),
+            ('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"),
+        ]
+
+        return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl")
+    else:
+        return ''
+
+
 def show_backtest_results(config: Dict, backtest_stats: Dict):
     stake_currency = config['stake_currency']
 
@@ -312,15 +432,21 @@ def show_backtest_results(config: Dict, backtest_stats: Dict):
 
         table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
                                        stake_currency=stake_currency)
-        if isinstance(table, str):
+        if isinstance(table, str) and len(table) > 0:
             print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
         print(table)
 
         table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
-        if isinstance(table, str):
+        if isinstance(table, str) and len(table) > 0:
             print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
         print(table)
-        if isinstance(table, str):
+
+        table = text_table_add_metrics(results)
+        if isinstance(table, str) and len(table) > 0:
+            print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
+        print(table)
+
+        if isinstance(table, str) and len(table) > 0:
             print('=' * len(table.splitlines()[0]))
         print()
 

freqtrade/pairlist/AgeFilter.py

@@ -26,12 +26,11 @@ class AgeFilter(IPairList):
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
 
         if self._min_days_listed < 1:
-            raise OperationalException("AgeFilter requires min_days_listed must be >= 1")
+            raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
         if self._min_days_listed > exchange.ohlcv_candle_limit:
-            raise OperationalException("AgeFilter requires min_days_listed must not exceed "
+            raise OperationalException("AgeFilter requires min_days_listed to not exceed "
                                        "exchange max request size "
                                        f"({exchange.ohlcv_candle_limit})")
-        self._enabled = self._min_days_listed >= 1
 
     @property
     def needstickers(self) -> bool:

freqtrade/pairlist/IPairList.py

@@ -162,6 +162,11 @@ class IPairList(ABC):
                                f"{self._exchange.name}. Removing it from whitelist..")
                 continue
 
+            if not self._exchange.market_is_tradable(markets[pair]):
+                logger.warning(f"Pair {pair} is not tradable with Freqtrade."
+                               "Removing it from whitelist..")
+                continue
+
             if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
                 logger.warning(f"Pair {pair} is not compatible with your stake currency "
                                f"{self._config['stake_currency']}. Removing it from whitelist..")

freqtrade/pairlist/PriceFilter.py

@@ -4,6 +4,7 @@ Price pair list filter
 import logging
 from typing import Any, Dict
 
+from freqtrade.exceptions import OperationalException
 from freqtrade.pairlist.IPairList import IPairList
 
 
@@ -18,11 +19,17 @@ class PriceFilter(IPairList):
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
         self._low_price_ratio = pairlistconfig.get('low_price_ratio', 0)
+        if self._low_price_ratio < 0:
+            raise OperationalException("PriceFilter requires low_price_ratio to be >= 0")
         self._min_price = pairlistconfig.get('min_price', 0)
+        if self._min_price < 0:
+            raise OperationalException("PriceFilter requires min_price to be >= 0")
         self._max_price = pairlistconfig.get('max_price', 0)
-        self._enabled = ((self._low_price_ratio != 0) or
-                         (self._min_price != 0) or
-                         (self._max_price != 0))
+        if self._max_price < 0:
+            raise OperationalException("PriceFilter requires max_price to be >= 0")
+        self._enabled = ((self._low_price_ratio > 0) or
+                         (self._min_price > 0) or
+                         (self._max_price > 0))
 
     @property
     def needstickers(self) -> bool:

freqtrade/pairlist/VolumePairList.py

@@ -14,7 +14,7 @@ from freqtrade.pairlist.IPairList import IPairList
 logger = logging.getLogger(__name__)
 
 
-SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume']
+SORT_VALUES = ['quoteVolume']
 
 
 class VolumePairList(IPairList):
@@ -45,11 +45,6 @@ class VolumePairList(IPairList):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
 
-        if self._sort_key != 'quoteVolume':
-            logger.warning(
-                "DEPRECATED: using any key other than quoteVolume for VolumePairList is deprecated."
-                )
-
     @property
     def needstickers(self) -> bool:
         """

freqtrade/persistence/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa: F401
+
+from freqtrade.persistence.models import (Order, Trade, clean_dry_run_db,
+                                          cleanup, init)

freqtrade/persistence/migrations.py

@@ -0,0 +1,149 @@
+import logging
+from typing import List
+
+from sqlalchemy import inspect
+
+logger = logging.getLogger(__name__)
+
+
+def get_table_names_for_table(inspector, tabletype):
+    return [t for t in inspector.get_table_names() if t.startswith(tabletype)]
+
+
+def has_column(columns: List, searchname: str) -> bool:
+    return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
+
+
+def get_column_def(columns: List, column: str, default: str) -> str:
+    return default if not has_column(columns, column) else column
+
+
+def get_backup_name(tabs, backup_prefix: str):
+    table_back_name = backup_prefix
+    for i, table_back_name in enumerate(tabs):
+        table_back_name = f'{backup_prefix}{i}'
+        logger.debug(f'trying {table_back_name}')
+
+    return table_back_name
+
+
+def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List):
+    fee_open = get_column_def(cols, 'fee_open', 'fee')
+    fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
+    fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
+    fee_close = get_column_def(cols, 'fee_close', 'fee')
+    fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
+    fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
+    open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
+    close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
+    stop_loss = get_column_def(cols, 'stop_loss', '0.0')
+    stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
+    initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
+    initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
+    stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
+    stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
+    max_rate = get_column_def(cols, 'max_rate', '0.0')
+    min_rate = get_column_def(cols, 'min_rate', 'null')
+    sell_reason = get_column_def(cols, 'sell_reason', 'null')
+    strategy = get_column_def(cols, 'strategy', 'null')
+    # If ticker-interval existed use that, else null.
+    if has_column(cols, 'ticker_interval'):
+        timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
+    else:
+        timeframe = get_column_def(cols, 'timeframe', 'null')
+
+    open_trade_price = get_column_def(cols, 'open_trade_price',
+                                      f'amount * open_rate * (1 + {fee_open})')
+    close_profit_abs = get_column_def(
+        cols, 'close_profit_abs',
+        f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
+    sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
+    amount_requested = get_column_def(cols, 'amount_requested', 'amount')
+
+    # Schema migration necessary
+    engine.execute(f"alter table trades rename to {table_back_name}")
+    # drop indexes on backup table
+    for index in inspector.get_indexes(table_back_name):
+        engine.execute(f"drop index {index['name']}")
+    # let SQLAlchemy create the schema as required
+    decl_base.metadata.create_all(engine)
+
+    # Copy data back - following the correct schema
+    engine.execute(f"""insert into trades
+            (id, exchange, pair, is_open,
+            fee_open, fee_open_cost, fee_open_currency,
+            fee_close, fee_close_cost, fee_open_currency, open_rate,
+            open_rate_requested, close_rate, close_rate_requested, close_profit,
+            stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
+            stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
+            stoploss_order_id, stoploss_last_update,
+            max_rate, min_rate, sell_reason, sell_order_status, strategy,
+            timeframe, open_trade_price, close_profit_abs
+            )
+        select id, lower(exchange),
+            case
+                when instr(pair, '_') != 0 then
+                substr(pair,    instr(pair, '_') + 1) || '/' ||
+                substr(pair, 1, instr(pair, '_') - 1)
+                else pair
+                end
+            pair,
+            is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
+            {fee_open_currency} fee_open_currency, {fee_close} fee_close,
+            {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
+            open_rate, {open_rate_requested} open_rate_requested, close_rate,
+            {close_rate_requested} close_rate_requested, close_profit,
+            stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
+            {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
+            {initial_stop_loss} initial_stop_loss,
+            {initial_stop_loss_pct} initial_stop_loss_pct,
+            {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
+            {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
+            {sell_order_status} sell_order_status,
+            {strategy} strategy, {timeframe} timeframe,
+            {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
+            from {table_back_name}
+            """)
+
+
+def migrate_open_orders_to_trades(engine):
+    engine.execute("""
+        insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open)
+        select id ft_trade_id, pair ft_pair, open_order_id,
+            case when close_rate_requested is null then 'buy'
+            else 'sell' end ft_order_side, 1 ft_is_open
+        from trades
+        where open_order_id is not null
+        union all
+        select id ft_trade_id, pair ft_pair, stoploss_order_id order_id,
+            'stoploss' ft_order_side, 1 ft_is_open
+        from trades
+        where stoploss_order_id is not null
+        """)
+
+
+def check_migrate(engine, decl_base, previous_tables) -> None:
+    """
+    Checks if migration is necessary and migrates if necessary
+    """
+    inspector = inspect(engine)
+
+    cols = inspector.get_columns('trades')
+    tabs = get_table_names_for_table(inspector, 'trades')
+    table_back_name = get_backup_name(tabs, 'trades_bak')
+
+    # Check for latest column
+    if not has_column(cols, 'amount_requested'):
+        logger.info(f'Running database migration for trades - backup: {table_back_name}')
+        migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
+        # Reread columns - the above recreated the table!
+        inspector = inspect(engine)
+        cols = inspector.get_columns('trades')
+
+    if 'orders' not in previous_tables:
+        logger.info('Moving open orders to Orders table.')
+        migrate_open_orders_to_trades(engine)
+    else:
+        pass
+        # Empty for now - as there is only one iteration of the orders table so far.
+        # table_back_name = get_backup_name(tabs, 'orders_bak')

freqtrade/persistence/models.py

@@ -2,21 +2,24 @@
 This module contains the class to persist trades into SQLite
 """
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from decimal import Decimal
 from typing import Any, Dict, List, Optional
 
 import arrow
-from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String,
-                        create_engine, desc, func, inspect)
+from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer,
+                        String, create_engine, desc, func, inspect)
 from sqlalchemy.exc import NoSuchModuleError
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import Query
+from sqlalchemy.orm import Query, relationship
 from sqlalchemy.orm.scoping import scoped_session
 from sqlalchemy.orm.session import sessionmaker
 from sqlalchemy.pool import StaticPool
+from sqlalchemy.sql.schema import UniqueConstraint
 
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import DependencyException, OperationalException
+from freqtrade.misc import safe_value_fallback
+from freqtrade.persistence.migrations import check_migrate
 
 logger = logging.getLogger(__name__)
 
@@ -56,120 +59,18 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
     # We should use the scoped_session object - not a seperately initialized version
     Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
     Trade.query = Trade.session.query_property()
+    # Copy session attributes to order object too
+    Order.session = Trade.session
+    Order.query = Order.session.query_property()
+    previous_tables = inspect(engine).get_table_names()
     _DECL_BASE.metadata.create_all(engine)
-    check_migrate(engine)
+    check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
 
     # Clean dry_run DB if the db is not in-memory
     if clean_open_orders and db_url != 'sqlite://':
         clean_dry_run_db()
 
 
-def has_column(columns: List, searchname: str) -> bool:
-    return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
-
-
-def get_column_def(columns: List, column: str, default: str) -> str:
-    return default if not has_column(columns, column) else column
-
-
-def check_migrate(engine) -> None:
-    """
-    Checks if migration is necessary and migrates if necessary
-    """
-    inspector = inspect(engine)
-
-    cols = inspector.get_columns('trades')
-    tabs = inspector.get_table_names()
-    table_back_name = 'trades_bak'
-    for i, table_back_name in enumerate(tabs):
-        table_back_name = f'trades_bak{i}'
-        logger.debug(f'trying {table_back_name}')
-
-    # Check for latest column
-    if not has_column(cols, 'timeframe'):
-        logger.info(f'Running database migration - backup available as {table_back_name}')
-
-        fee_open = get_column_def(cols, 'fee_open', 'fee')
-        fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
-        fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
-        fee_close = get_column_def(cols, 'fee_close', 'fee')
-        fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
-        fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
-        open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
-        close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
-        stop_loss = get_column_def(cols, 'stop_loss', '0.0')
-        stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
-        initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
-        initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
-        stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
-        stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
-        max_rate = get_column_def(cols, 'max_rate', '0.0')
-        min_rate = get_column_def(cols, 'min_rate', 'null')
-        sell_reason = get_column_def(cols, 'sell_reason', 'null')
-        strategy = get_column_def(cols, 'strategy', 'null')
-        # If ticker-interval existed use that, else null.
-        if has_column(cols, 'ticker_interval'):
-            timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
-        else:
-            timeframe = get_column_def(cols, 'timeframe', 'null')
-
-        open_trade_price = get_column_def(cols, 'open_trade_price',
-                                          f'amount * open_rate * (1 + {fee_open})')
-        close_profit_abs = get_column_def(
-            cols, 'close_profit_abs',
-            f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
-        sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
-
-        # Schema migration necessary
-        engine.execute(f"alter table trades rename to {table_back_name}")
-        # drop indexes on backup table
-        for index in inspector.get_indexes(table_back_name):
-            engine.execute(f"drop index {index['name']}")
-        # let SQLAlchemy create the schema as required
-        _DECL_BASE.metadata.create_all(engine)
-
-        # Copy data back - following the correct schema
-        engine.execute(f"""insert into trades
-                (id, exchange, pair, is_open,
-                fee_open, fee_open_cost, fee_open_currency,
-                fee_close, fee_close_cost, fee_open_currency, open_rate,
-                open_rate_requested, close_rate, close_rate_requested, close_profit,
-                stake_amount, amount, open_date, close_date, open_order_id,
-                stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
-                stoploss_order_id, stoploss_last_update,
-                max_rate, min_rate, sell_reason, sell_order_status, strategy,
-                timeframe, open_trade_price, close_profit_abs
-                )
-            select id, lower(exchange),
-                case
-                    when instr(pair, '_') != 0 then
-                    substr(pair,    instr(pair, '_') + 1) || '/' ||
-                    substr(pair, 1, instr(pair, '_') - 1)
-                    else pair
-                    end
-                pair,
-                is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
-                {fee_open_currency} fee_open_currency, {fee_close} fee_close,
-                {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
-                open_rate, {open_rate_requested} open_rate_requested, close_rate,
-                {close_rate_requested} close_rate_requested, close_profit,
-                stake_amount, amount, open_date, close_date, open_order_id,
-                {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
-                {initial_stop_loss} initial_stop_loss,
-                {initial_stop_loss_pct} initial_stop_loss_pct,
-                {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
-                {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
-                {sell_order_status} sell_order_status,
-                {strategy} strategy, {timeframe} timeframe,
-                {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
-                from {table_back_name}
-             """)
-
-        # Reread columns - the above recreated the table!
-        inspector = inspect(engine)
-        cols = inspector.get_columns('trades')
-
-
 def cleanup() -> None:
     """
     Flushes all pending operations to disk.
@@ -189,13 +90,117 @@ def clean_dry_run_db() -> None:
             trade.open_order_id = None
 
 
+class Order(_DECL_BASE):
+    """
+    Order database model
+    Keeps a record of all orders placed on the exchange
+
+    One to many relationship with Trades:
+      - One trade can have many orders
+      - One Order can only be associated with one Trade
+
+    Mirrors CCXT Order structure
+    """
+    __tablename__ = 'orders'
+    # Uniqueness should be ensured over pair, order_id
+    # its likely that order_id is unique per Pair on some exchanges.
+    __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),)
+
+    id = Column(Integer, primary_key=True)
+    ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True)
+
+    trade = relationship("Trade", back_populates="orders")
+
+    ft_order_side = Column(String, nullable=False)
+    ft_pair = Column(String, nullable=False)
+    ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
+
+    order_id = Column(String, nullable=False, index=True)
+    status = Column(String, nullable=True)
+    symbol = Column(String, nullable=True)
+    order_type = Column(String, nullable=True)
+    side = Column(String, nullable=True)
+    price = Column(Float, nullable=True)
+    amount = Column(Float, nullable=True)
+    filled = Column(Float, nullable=True)
+    remaining = Column(Float, nullable=True)
+    cost = Column(Float, nullable=True)
+    order_date = Column(DateTime, nullable=True, default=datetime.utcnow)
+    order_filled_date = Column(DateTime, nullable=True)
+    order_update_date = Column(DateTime, nullable=True)
+
+    def __repr__(self):
+
+        return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, '
+                f'side={self.side}, order_type={self.order_type}, status={self.status})')
+
+    def update_from_ccxt_object(self, order):
+        """
+        Update Order from ccxt response
+        Only updates if fields are available from ccxt -
+        """
+        if self.order_id != str(order['id']):
+            raise DependencyException("Order-id's don't match")
+
+        self.status = order.get('status', self.status)
+        self.symbol = order.get('symbol', self.symbol)
+        self.order_type = order.get('type', self.order_type)
+        self.side = order.get('side', self.side)
+        self.price = order.get('price', self.price)
+        self.amount = order.get('amount', self.amount)
+        self.filled = order.get('filled', self.filled)
+        self.remaining = order.get('remaining', self.remaining)
+        self.cost = order.get('cost', self.cost)
+        if 'timestamp' in order and order['timestamp'] is not None:
+            self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
+
+        self.ft_is_open = True
+        if self.status in ('closed', 'canceled', 'cancelled'):
+            self.ft_is_open = False
+            if order.get('filled', 0) > 0:
+                self.order_filled_date = arrow.utcnow().datetime
+        self.order_update_date = arrow.utcnow().datetime
+
+    @staticmethod
+    def update_orders(orders: List['Order'], order: Dict[str, Any]):
+        """
+        Get all non-closed orders - useful when trying to batch-update orders
+        """
+        filtered_orders = [o for o in orders if o.order_id == order['id']]
+        if filtered_orders:
+            oobj = filtered_orders[0]
+            oobj.update_from_ccxt_object(order)
+        else:
+            logger.warning(f"Did not find order for {order['id']}.")
+
+    @staticmethod
+    def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order':
+        """
+        Parse an order from a ccxt object and return a new order Object.
+        """
+        o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair)
+
+        o.update_from_ccxt_object(order)
+        return o
+
+    @staticmethod
+    def get_open_orders() -> List['Order']:
+        """
+        """
+        return Order.query.filter(Order.ft_is_open.is_(True)).all()
+
+
 class Trade(_DECL_BASE):
     """
-    Class used to define a trade structure
+    Trade database model.
+    Also handles updating and querying trades
     """
     __tablename__ = 'trades'
 
     id = Column(Integer, primary_key=True)
+
+    orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan")
+
     exchange = Column(String, nullable=False)
     pair = Column(String, nullable=False, index=True)
     is_open = Column(Boolean, nullable=False, default=True, index=True)
@@ -215,6 +220,7 @@ class Trade(_DECL_BASE):
     close_profit_abs = Column(Float)
     stake_amount = Column(Float, nullable=False)
     amount = Column(Float)
+    amount_requested = Column(Float)
     open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
     close_date = Column(DateTime)
     open_order_id = Column(String)
@@ -256,6 +262,7 @@ class Trade(_DECL_BASE):
             'is_open': self.is_open,
             'exchange': self.exchange,
             'amount': round(self.amount, 8),
+            'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None,
             'stake_amount': round(self.stake_amount, 8),
             'strategy': self.strategy,
             'ticker_interval': self.timeframe,  # DEPRECATED
@@ -270,16 +277,17 @@ class Trade(_DECL_BASE):
 
             'open_date_hum': arrow.get(self.open_date).humanize(),
             'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"),
-            'open_timestamp': int(self.open_date.timestamp() * 1000),
+            'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
             'open_rate': self.open_rate,
             'open_rate_requested': self.open_rate_requested,
-            'open_trade_price': self.open_trade_price,
+            'open_trade_price': round(self.open_trade_price, 8),
 
             'close_date_hum': (arrow.get(self.close_date).humanize()
                                if self.close_date else None),
             'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S")
                            if self.close_date else None),
-            'close_timestamp': int(self.close_date.timestamp() * 1000) if self.close_date else None,
+            'close_timestamp': int(self.close_date.replace(
+                tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None,
             'close_rate': self.close_rate,
             'close_rate_requested': self.close_rate_requested,
             'close_profit': self.close_profit,
@@ -294,8 +302,8 @@ class Trade(_DECL_BASE):
             'stoploss_order_id': self.stoploss_order_id,
             'stoploss_last_update': (self.stoploss_last_update.strftime("%Y-%m-%d %H:%M:%S")
                                      if self.stoploss_last_update else None),
-            'stoploss_last_update_timestamp': (int(self.stoploss_last_update.timestamp() * 1000)
-                                               if self.stoploss_last_update else None),
+            'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace(
+                tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None,
             'initial_stop_loss': self.initial_stop_loss,  # Deprecated - should not be used
             'initial_stop_loss_abs': self.initial_stop_loss,
             'initial_stop_loss_ratio': (self.initial_stop_loss_pct
@@ -365,25 +373,28 @@ class Trade(_DECL_BASE):
         """
         order_type = order['type']
         # Ignore open and cancelled orders
-        if order['status'] == 'open' or order['price'] is None:
+        if order['status'] == 'open' or safe_value_fallback(order, 'average', 'price') is None:
             return
 
         logger.info('Updating trade (id=%s) ...', self.id)
 
         if order_type in ('market', 'limit') and order['side'] == 'buy':
             # Update open rate and actual amount
-            self.open_rate = Decimal(order['price'])
-            self.amount = Decimal(order.get('filled', order['amount']))
+            self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price'))
+            self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount'))
             self.recalc_open_trade_price()
-            logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self)
+            if self.is_open:
+                logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.')
             self.open_order_id = None
         elif order_type in ('market', 'limit') and order['side'] == 'sell':
-            self.close(order['price'])
-            logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self)
+            if self.is_open:
+                logger.info(f'{order_type.upper()}_SELL has been fulfilled for {self}.')
+            self.close(safe_value_fallback(order, 'average', 'price'))
         elif order_type in ('stop_loss_limit', 'stop-loss', 'stop'):
             self.stoploss_order_id = None
             self.close_rate_requested = self.stop_loss
-            logger.info('%s is hit for %s.', order_type.upper(), self)
+            if self.is_open:
+                logger.info(f'{order_type.upper()} is hit for {self}.')
             self.close(order['average'])
         else:
             raise ValueError(f'Unknown order type: {order_type}')
@@ -397,7 +408,7 @@ class Trade(_DECL_BASE):
         self.close_rate = Decimal(rate)
         self.close_profit = self.calc_profit_ratio()
         self.close_profit_abs = self.calc_profit()
-        self.close_date = datetime.utcnow()
+        self.close_date = self.close_date or datetime.utcnow()
         self.is_open = False
         self.sell_order_status = 'closed'
         self.open_order_id = None
@@ -435,6 +446,17 @@ class Trade(_DECL_BASE):
         else:
             return False
 
+    def update_order(self, order: Dict) -> None:
+        Order.update_orders(self.orders, order)
+
+    def delete(self) -> None:
+
+        for order in self.orders:
+            Order.session.delete(order)
+
+        Trade.session.delete(self)
+        Trade.session.flush()
+
     def _calc_open_trade_price(self) -> float:
         """
         Calculate the open_rate including open_fee.
@@ -501,6 +523,21 @@ class Trade(_DECL_BASE):
         profit_ratio = (close_trade_price / self.open_trade_price) - 1
         return float(f"{profit_ratio:.8f}")
 
+    def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]:
+        """
+        Finds latest order for this orderside and status
+        :param order_side: Side of the order (either 'buy' or 'sell')
+        :param is_open: Only search for open orders?
+        :return: latest Order object if it exists, else None
+        """
+        orders = [o for o in self.orders if o.side == order_side]
+        if is_open is not None:
+            orders = [o for o in orders if o.ft_is_open == is_open]
+        if len(orders) > 0:
+            return orders[-1]
+        else:
+            return None
+
     @staticmethod
     def get_trades(trade_filter=None) -> Query:
         """
@@ -532,6 +569,26 @@ class Trade(_DECL_BASE):
         """
         return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
 
+    @staticmethod
+    def get_open_trades_without_assigned_fees():
+        """
+        Returns all open trades which don't have open fees set correctly
+        """
+        return Trade.get_trades([Trade.fee_open_currency.is_(None),
+                                 Trade.orders.any(),
+                                 Trade.is_open.is_(True),
+                                 ]).all()
+
+    @staticmethod
+    def get_sold_trades_without_assigned_fees():
+        """
+        Returns all closed trades which don't have fees set correctly
+        """
+        return Trade.get_trades([Trade.fee_close_currency.is_(None),
+                                 Trade.orders.any(),
+                                 Trade.is_open.is_(False),
+                                 ]).all()
+
     @staticmethod
     def total_open_trades_stakes() -> float:
         """

freqtrade/plot/plotting.py

@@ -8,7 +8,8 @@ from freqtrade.configuration import TimeRange
 from freqtrade.data.btanalysis import (calculate_max_drawdown,
                                        combine_dataframes_with_mean,
                                        create_cum_profit,
-                                       extract_trades_of_period, load_trades)
+                                       extract_trades_of_period,
+                                       load_trades)
 from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history import load_data
@@ -37,15 +38,15 @@ def init_plotscript(config):
     """
 
     if "pairs" in config:
-        pairs = config["pairs"]
+        pairs = config['pairs']
     else:
-        pairs = config["exchange"]["pair_whitelist"]
+        pairs = config['exchange']['pair_whitelist']
 
     # Set timerange to use
-    timerange = TimeRange.parse_timerange(config.get("timerange"))
+    timerange = TimeRange.parse_timerange(config.get('timerange'))
 
     data = load_data(
-        datadir=config.get("datadir"),
+        datadir=config.get('datadir'),
         pairs=pairs,
         timeframe=config.get('timeframe', '5m'),
         timerange=timerange,
@@ -53,19 +54,22 @@ def init_plotscript(config):
     )
 
     no_trades = False
+    filename = config.get('exportfilename')
     if config.get('no_trades', False):
         no_trades = True
-    elif not config['exportfilename'].is_file() and config['trade_source'] == 'file':
+    elif config['trade_source'] == 'file':
+        if not filename.is_dir() and not filename.is_file():
             logger.warning("Backtest file is missing skipping trades.")
             no_trades = True
 
     trades = load_trades(
         config['trade_source'],
         db_url=config.get('db_url'),
-        exportfilename=config.get('exportfilename'),
-        no_trades=no_trades
+        exportfilename=filename,
+        no_trades=no_trades,
+        strategy=config.get('strategy'),
     )
-    trades = trim_dataframe(trades, timerange, 'open_time')
+    trades = trim_dataframe(trades, timerange, 'open_date')
 
     return {"ohlcv": data,
             "trades": trades,
@@ -165,10 +169,11 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
     if trades is not None and len(trades) > 0:
         # Create description for sell summarizing the trade
         trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, "
-                                                  f"{row['sell_reason']}, {row['duration']} min",
+                                                  f"{row['sell_reason']}, "
+                                                  f"{row['trade_duration']} min",
                                                   axis=1)
         trade_buys = go.Scatter(
-            x=trades["open_time"],
+            x=trades["open_date"],
             y=trades["open_rate"],
             mode='markers',
             name='Trade buy',
@@ -183,7 +188,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
         )
 
         trade_sells = go.Scatter(
-            x=trades.loc[trades['profit_percent'] > 0, "close_time"],
+            x=trades.loc[trades['profit_percent'] > 0, "close_date"],
             y=trades.loc[trades['profit_percent'] > 0, "close_rate"],
             text=trades.loc[trades['profit_percent'] > 0, "desc"],
             mode='markers',
@@ -196,7 +201,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
             )
         )
         trade_sells_loss = go.Scatter(
-            x=trades.loc[trades['profit_percent'] <= 0, "close_time"],
+            x=trades.loc[trades['profit_percent'] <= 0, "close_date"],
             y=trades.loc[trades['profit_percent'] <= 0, "close_rate"],
             text=trades.loc[trades['profit_percent'] <= 0, "desc"],
             mode='markers',
@@ -486,13 +491,13 @@ def load_and_plot_trades(config: Dict[str, Any]):
             pair=pair,
             data=df_analyzed,
             trades=trades_pair,
-            indicators1=config.get("indicators1", []),
-            indicators2=config.get("indicators2", []),
+            indicators1=config.get('indicators1', []),
+            indicators2=config.get('indicators2', []),
             plot_config=strategy.plot_config if hasattr(strategy, 'plot_config') else {}
         )
 
         store_plot_file(fig, filename=generate_plot_filename(pair, config['timeframe']),
-                        directory=config['user_data_dir'] / "plot")
+                        directory=config['user_data_dir'] / 'plot')
 
     logger.info('End of plotting process. %s plots generated', pair_counter)
 
@@ -509,8 +514,8 @@ def plot_profit(config: Dict[str, Any]) -> None:
     # Filter trades to relevant pairs
     # Remove open pairs - we don't know the profit yet so can't calculate profit for these.
     # Also, If only one open pair is left, then the profit-generation would fail.
-    trades = trades[(trades['pair'].isin(plot_elements["pairs"]))
-                    & (~trades['close_time'].isnull())
+    trades = trades[(trades['pair'].isin(plot_elements['pairs']))
+                    & (~trades['close_date'].isnull())
                     ]
     if len(trades) == 0:
         raise OperationalException("No trades found, cannot generate Profit-plot without "
@@ -518,7 +523,7 @@ def plot_profit(config: Dict[str, Any]) -> None:
 
     # Create an average close price of all the pairs that were involved.
     # this could be useful to gauge the overall market trend
-    fig = generate_profit_graph(plot_elements["pairs"], plot_elements["ohlcv"],
+    fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'],
                                 trades, config.get('timeframe', '5m'))
     store_plot_file(fig, filename='freqtrade-profit-plot.html',
-                    directory=config['user_data_dir'] / "plot", auto_open=True)
+                    directory=config['user_data_dir'] / 'plot', auto_open=True)

freqtrade/resolvers/hyperopt_resolver.py

@@ -23,7 +23,7 @@ class HyperOptResolver(IResolver):
     object_type = IHyperOpt
     object_type_str = "Hyperopt"
     user_subdir = USERPATH_HYPEROPTS
-    initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve()
+    initial_search_path = None
 
     @staticmethod
     def load_hyperopt(config: Dict) -> IHyperOpt:

freqtrade/resolvers/iresolver.py

@@ -59,7 +59,7 @@ class IResolver:
         module = importlib.util.module_from_spec(spec)
         try:
             spec.loader.exec_module(module)  # type: ignore # importlib does not use typehints
-        except (ModuleNotFoundError, SyntaxError) as err:
+        except (ModuleNotFoundError, SyntaxError, ImportError) as err:
             # Catch errors in case a specific module is not installed
             logger.warning(f"Could not import {module_path} due to '{err}'")
             if enum_failed:

freqtrade/rpc/api_server.py

@@ -16,8 +16,10 @@ from werkzeug.security import safe_str_cmp
 from werkzeug.serving import make_server
 
 from freqtrade.__init__ import __version__
-from freqtrade.rpc.rpc import RPC, RPCException
+from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.persistence import Trade
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
+from freqtrade.rpc.rpc import RPC, RPCException
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +34,7 @@ class ArrowJSONEncoder(JSONEncoder):
             elif isinstance(obj, date):
                 return obj.strftime("%Y-%m-%d")
             elif isinstance(obj, datetime):
-                return obj.strftime("%Y-%m-%d %H:%M:%S")
+                return obj.strftime(DATETIME_PRINT_FORMAT)
             iterable = iter(obj)
         except TypeError:
             pass
@@ -56,7 +58,7 @@ def require_login(func: Callable[[Any, Any], Any]):
 
 
 # Type should really be Callable[[ApiServer], Any], but that will create a circular dependency
-def rpc_catch_errors(func: Callable[[Any], Any]):
+def rpc_catch_errors(func: Callable[..., Any]):
 
     def func_wrapper(obj, *args, **kwargs):
 
@@ -69,6 +71,11 @@ def rpc_catch_errors(func: Callable[[Any], Any]):
     return func_wrapper
 
 
+def shutdown_session(exception=None):
+    # Remove scoped session
+    Trade.session.remove()
+
+
 class ApiServer(RPC):
     """
     This class runs api server and provides rpc.rpc functionality to it
@@ -103,6 +110,8 @@ class ApiServer(RPC):
         self.jwt = JWTManager(self.app)
         self.app.json_encoder = ArrowJSONEncoder
 
+        self.app.teardown_appcontext(shutdown_session)
+
         # Register application handling
         self.register_rest_rpc_urls()
 
@@ -186,6 +195,7 @@ class ApiServer(RPC):
         self.app.add_url_rule(f'{BASE_URI}/count', 'count', view_func=self._count, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/daily', 'daily', view_func=self._daily, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/edge', 'edge', view_func=self._edge, methods=['GET'])
+        self.app.add_url_rule(f'{BASE_URI}/logs', 'log', view_func=self._get_logs, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/profit', 'profit',
                               view_func=self._profit, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/performance', 'performance',
@@ -200,6 +210,8 @@ class ApiServer(RPC):
                               view_func=self._ping, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/trades', 'trades',
                               view_func=self._trades, methods=['GET'])
+        self.app.add_url_rule(f'{BASE_URI}/trades/<int:tradeid>', 'trades_delete',
+                              view_func=self._trades_delete, methods=['DELETE'])
         # Combined actions and infos
         self.app.add_url_rule(f'{BASE_URI}/blacklist', 'blacklist', view_func=self._blacklist,
                               methods=['GET', 'POST'])
@@ -210,9 +222,6 @@ class ApiServer(RPC):
         self.app.add_url_rule(f'{BASE_URI}/forcesell', 'forcesell', view_func=self._forcesell,
                               methods=['POST'])
 
-        # TODO: Implement the following
-        # help (?)
-
     @require_login
     def page_not_found(self, error):
         """
@@ -346,6 +355,18 @@ class ApiServer(RPC):
 
         return self.rest_dump(stats)
 
+    @require_login
+    @rpc_catch_errors
+    def _get_logs(self):
+        """
+        Returns latest logs
+         get:
+          param:
+            limit: Only get a certain number of records
+        """
+        limit = int(request.args.get('limit', 0)) or None
+        return self.rest_dump(self._rpc_get_logs(limit))
+
     @require_login
     @rpc_catch_errors
     def _edge(self):
@@ -424,6 +445,19 @@ class ApiServer(RPC):
         results = self._rpc_trade_history(limit)
         return self.rest_dump(results)
 
+    @require_login
+    @rpc_catch_errors
+    def _trades_delete(self, tradeid):
+        """
+        Handler for DELETE /trades/<tradeid> endpoint.
+        Removes the trade from the database (tries to cancel open orders first!)
+        get:
+          param:
+            tradeid: Numeric trade-id assigned to the trade.
+        """
+        result = self._rpc_delete(tradeid)
+        return self.rest_dump(result)
+
     @require_login
     @rpc_catch_errors
     def _whitelist(self):

freqtrade/rpc/rpc.py

@@ -6,14 +6,15 @@ from abc import abstractmethod
 from datetime import date, datetime, timedelta
 from enum import Enum
 from math import isnan
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import arrow
 from numpy import NAN, mean
 
+from freqtrade.constants import CANCEL_REASON
 from freqtrade.exceptions import ExchangeError, PricingError
-
-from freqtrade.exchange import timeframe_to_msecs, timeframe_to_minutes
+from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
+from freqtrade.loggers import bufferHandler
 from freqtrade.misc import shorten_date
 from freqtrade.persistence import Trade
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -26,7 +27,7 @@ logger = logging.getLogger(__name__)
 class RPCMessageType(Enum):
     STATUS_NOTIFICATION = 'status'
     WARNING_NOTIFICATION = 'warning'
-    CUSTOM_NOTIFICATION = 'custom'
+    STARTUP_NOTIFICATION = 'startup'
     BUY_NOTIFICATION = 'buy'
     BUY_CANCEL_NOTIFICATION = 'buy_cancel'
     SELL_NOTIFICATION = 'sell'
@@ -35,6 +36,9 @@ class RPCMessageType(Enum):
     def __repr__(self):
         return self.value
 
+    def __str__(self):
+        return self.value
+
 
 class RPCException(Exception):
     """
@@ -158,6 +162,7 @@ class RPC:
                     current_profit_abs=current_profit_abs,
                     stoploss_current_dist=stoploss_current_dist,
                     stoploss_current_dist_ratio=round(stoploss_current_dist_ratio, 8),
+                    stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2),
                     stoploss_entry_dist=stoploss_entry_dist,
                     stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8),
                     open_order='({} {} rem={:.8f})'.format(
@@ -222,24 +227,23 @@ class RPC:
                 Trade.close_date >= profitday,
                 Trade.close_date < (profitday + timedelta(days=1))
             ]).order_by(Trade.close_date).all()
-            curdayprofit = sum(trade.close_profit_abs for trade in trades)
+            curdayprofit = sum(
+                trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None)
             profit_days[profitday] = {
-                'amount': f'{curdayprofit:.8f}',
+                'amount': curdayprofit,
                 'trades': len(trades)
             }
 
         data = [
             {
                 'date': key,
-                'abs_profit': f'{float(value["amount"]):.8f}',
-                'fiat_value': '{value:.3f}'.format(
-                    value=self._fiat_converter.convert_amount(
+                'abs_profit': value["amount"],
+                'fiat_value': self._fiat_converter.convert_amount(
                         value['amount'],
                         stake_currency,
                         fiat_display_currency
                     ) if self._fiat_converter else 0,
-                ),
-                'trade_count': f'{value["trades"]}',
+                'trade_count': value["trades"],
             }
             for key, value in profit_days.items()
         ]
@@ -435,7 +439,7 @@ class RPC:
     def _rpc_reload_config(self) -> Dict[str, str]:
         """ Handler for reload_config. """
         self._freqtrade.state = State.RELOAD_CONFIG
-        return {'status': 'reloading config ...'}
+        return {'status': 'Reloading config ...'}
 
     def _rpc_stopbuy(self) -> Dict[str, str]:
         """
@@ -454,26 +458,19 @@ class RPC:
         """
         def _exec_forcesell(trade: Trade) -> None:
             # Check if there is there is an open order
+            fully_canceled = False
             if trade.open_order_id:
                 order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
 
-                # Cancel open LIMIT_BUY orders and close trade
-                if order and order['status'] == 'open' \
-                        and order['type'] == 'limit' \
-                        and order['side'] == 'buy':
-                    self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair)
-                    trade.close(order.get('price') or trade.open_rate)
-                    # Do the best effort, if we don't know 'filled' amount, don't try selling
-                    if order['filled'] is None:
-                        return
-                    trade.amount = order['filled']
+                if order['side'] == 'buy':
+                    fully_canceled = self._freqtrade.handle_cancel_buy(
+                        trade, order, CANCEL_REASON['FORCE_SELL'])
 
-                # Ignore trades with an attached LIMIT_SELL order
-                if order and order['status'] == 'open' \
-                        and order['type'] == 'limit' \
-                        and order['side'] == 'sell':
-                    return
+                if order['side'] == 'sell':
+                    # Cancel order - so it is placed anew with a fresh price.
+                    self._freqtrade.handle_cancel_sell(trade, order, CANCEL_REASON['FORCE_SELL'])
 
+            if not fully_canceled:
                 # Get current rate and execute sell
                 current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
                 self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL)
@@ -538,6 +535,45 @@ class RPC:
         else:
             return None
 
+    def _rpc_delete(self, trade_id: str) -> Dict[str, Union[str, int]]:
+        """
+        Handler for delete <id>.
+        Delete the given trade and close eventually existing open orders.
+        """
+        with self._freqtrade._sell_lock:
+            c_count = 0
+            trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first()
+            if not trade:
+                logger.warning('delete trade: Invalid argument received')
+                raise RPCException('invalid argument')
+
+            # Try cancelling regular order if that exists
+            if trade.open_order_id:
+                try:
+                    self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair)
+                    c_count += 1
+                except (ExchangeError):
+                    pass
+
+            # cancel stoploss on exchange ...
+            if (self._freqtrade.strategy.order_types.get('stoploss_on_exchange')
+                    and trade.stoploss_order_id):
+                try:
+                    self._freqtrade.exchange.cancel_stoploss_order(trade.stoploss_order_id,
+                                                                   trade.pair)
+                    c_count += 1
+                except (ExchangeError):
+                    pass
+
+            trade.delete()
+            self._freqtrade.wallets.update()
+            return {
+                'result': 'success',
+                'trade_id': trade_id,
+                'result_msg': f'Deleted trade {trade_id}. Closed {c_count} open orders.',
+                'cancel_order_count': c_count,
+            }
+
     def _rpc_performance(self) -> List[Dict[str, Any]]:
         """
         Handler for performance.
@@ -593,6 +629,24 @@ class RPC:
                }
         return res
 
+    def _rpc_get_logs(self, limit: Optional[int]) -> Dict[str, Any]:
+        """Returns the last X logs"""
+        if limit:
+            buffer = bufferHandler.buffer[-limit:]
+        else:
+            buffer = bufferHandler.buffer
+        records = [[datetime.fromtimestamp(r.created).strftime("%Y-%m-%d %H:%M:%S"),
+                   r.created * 1000, r.name, r.levelname,
+                   r.message + ('\n' + r.exc_text if r.exc_text else '')]
+                   for r in buffer]
+
+        # Log format:
+        # [logtime-formatted, logepoch, logger-name, loglevel, message \n + exception]
+        # e.g. ["2020-08-27 11:35:01", 1598520901097.9397,
+        #       "freqtrade.worker", "INFO", "Starting worker develop"]
+
+        return {'log_count': len(records), 'logs': records}
+
     def _rpc_edge(self) -> List[Dict[str, Any]]:
         """ Returns information related to Edge """
         if not self._freqtrade.edge:

freqtrade/rpc/rpc_manager.py

@@ -59,7 +59,7 @@ class RPCManager:
             try:
                 mod.send_msg(msg)
             except NotImplementedError:
-                logger.error(f"Message type {msg['type']} not implemented by handler {mod.name}.")
+                logger.error(f"Message type '{msg['type']}' not implemented by handler {mod.name}.")
 
     def startup_messages(self, config: Dict[str, Any], pairlist) -> None:
         if config['dry_run']:
@@ -76,7 +76,7 @@ class RPCManager:
         exchange_name = config['exchange']['name']
         strategy_name = config.get('strategy', '')
         self.send_msg({
-            'type': RPCMessageType.CUSTOM_NOTIFICATION,
+            'type': RPCMessageType.STARTUP_NOTIFICATION,
             'status': f'*Exchange:* `{exchange_name}`\n'
                       f'*Stake per trade:* `{stake_amount} {stake_currency}`\n'
                       f'*Minimum ROI:* `{minimal_roi}`\n'
@@ -85,7 +85,7 @@ class RPCManager:
                       f'*Strategy:* `{strategy_name}`'
         })
         self.send_msg({
-            'type': RPCMessageType.STATUS_NOTIFICATION,
+            'type': RPCMessageType.STARTUP_NOTIFICATION,
             'status': f'Searching for {stake_currency} pairs to buy and sell '
                       f'based on {pairlist.short_desc()}'
         })

freqtrade/rpc/telegram.py

@@ -12,6 +12,7 @@ from tabulate import tabulate
 from telegram import ParseMode, ReplyKeyboardMarkup, Update
 from telegram.error import NetworkError, TelegramError
 from telegram.ext import CallbackContext, CommandHandler, Updater
+from telegram.utils.helpers import escape_markdown
 
 from freqtrade.__init__ import __version__
 from freqtrade.rpc import RPC, RPCException, RPCMessageType
@@ -94,6 +95,7 @@ class Telegram(RPC):
             CommandHandler('forcesell', self._forcesell),
             CommandHandler('forcebuy', self._forcebuy),
             CommandHandler('trades', self._trades),
+            CommandHandler('delete', self._delete_trade),
             CommandHandler('performance', self._performance),
             CommandHandler('daily', self._daily),
             CommandHandler('count', self._count),
@@ -102,6 +104,7 @@ class Telegram(RPC):
             CommandHandler('stopbuy', self._stopbuy),
             CommandHandler('whitelist', self._whitelist),
             CommandHandler('blacklist', self._blacklist),
+            CommandHandler('logs', self._logs),
             CommandHandler('edge', self._edge),
             CommandHandler('help', self._help),
             CommandHandler('version', self._version),
@@ -129,6 +132,13 @@ class Telegram(RPC):
     def send_msg(self, msg: Dict[str, Any]) -> None:
         """ Send a message to telegram channel """
 
+        noti = self._config['telegram'].get('notification_settings', {}
+                                            ).get(str(msg['type']), 'on')
+        if noti == 'off':
+            logger.info(f"Notification '{msg['type']}' not sent.")
+            # Notification disabled
+            return
+
         if msg['type'] == RPCMessageType.BUY_NOTIFICATION:
             if self._fiat_converter:
                 msg['stake_amount_fiat'] = self._fiat_converter.convert_amount(
@@ -148,7 +158,7 @@ class Telegram(RPC):
 
         elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION:
             message = ("\N{WARNING SIGN} *{exchange}:* "
-                       "Cancelling Open Buy Order for {pair}".format(**msg))
+                       "Cancelling open buy Order for {pair}. Reason: {reason}.".format(**msg))
 
         elif msg['type'] == RPCMessageType.SELL_NOTIFICATION:
             msg['amount'] = round(msg['amount'], 8)
@@ -187,13 +197,13 @@ class Telegram(RPC):
         elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION:
             message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
 
-        elif msg['type'] == RPCMessageType.CUSTOM_NOTIFICATION:
+        elif msg['type'] == RPCMessageType.STARTUP_NOTIFICATION:
             message = '{status}'.format(**msg)
 
         else:
             raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
 
-        self._send_msg(message)
+        self._send_msg(message, disable_notification=(noti == 'silent'))
 
     def _get_sell_emoji(self, msg):
         """
@@ -238,17 +248,18 @@ class Telegram(RPC):
                     ("*Close Profit:* `{close_profit_pct}`"
                      if r['close_profit_pct'] is not None else ""),
                     "*Current Profit:* `{current_profit_pct:.2f}%`",
-
+                ]
+                if (r['stop_loss'] != r['initial_stop_loss']
+                        and r['initial_stop_loss_pct'] is not None):
                     # Adding initial stoploss only if it is different from stoploss
-                    "*Initial Stoploss:* `{initial_stop_loss:.8f}` " +
-                    ("`({initial_stop_loss_pct:.2f}%)`") if (
-                            r['stop_loss'] != r['initial_stop_loss']
-                            and r['initial_stop_loss_pct'] is not None) else "",
+                    lines.append("*Initial Stoploss:* `{initial_stop_loss:.8f}` "
+                                 "`({initial_stop_loss_pct:.2f}%)`")
 
                 # Adding stoploss and stoploss percentage only if it is not None
-                    "*Stoploss:* `{stop_loss:.8f}` " +
-                    ("`({stop_loss_pct:.2f}%)`" if r['stop_loss_pct'] else ""),
-                ]
+                lines.append("*Stoploss:* `{stop_loss:.8f}` " +
+                             ("`({stop_loss_pct:.2f}%)`" if r['stop_loss_pct'] else ""))
+                lines.append("*Stoploss distance:* `{stoploss_current_dist:.8f}` "
+                             "`({stoploss_current_dist_pct:.2f}%)`")
                 if r['open_order']:
                     if r['sell_order_status']:
                         lines.append("*Open Order:* `{open_order}` - `{sell_order_status}`")
@@ -304,8 +315,8 @@ class Telegram(RPC):
             )
             stats_tab = tabulate(
                 [[day['date'],
-                  f"{day['abs_profit']} {stats['stake_currency']}",
-                  f"{day['fiat_value']} {stats['fiat_display_currency']}",
+                  f"{day['abs_profit']:.8f} {stats['stake_currency']}",
+                  f"{day['fiat_value']:.3f} {stats['fiat_display_currency']}",
                   f"{day['trade_count']} trades"] for day in stats['data']],
                 headers=[
                     'Day',
@@ -533,6 +544,27 @@ class Telegram(RPC):
         except RPCException as e:
             self._send_msg(str(e))
 
+    @authorized_only
+    def _delete_trade(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /delete <id>.
+        Delete the given trade
+        :param bot: telegram bot
+        :param update: message update
+        :return: None
+        """
+
+        trade_id = context.args[0] if len(context.args) > 0 else None
+        try:
+            msg = self._rpc_delete(trade_id)
+            self._send_msg((
+                '`{result_msg}`\n'
+                'Please make sure to take care of this asset on the exchange manually.'
+            ).format(**msg))
+
+        except RPCException as e:
+            self._send_msg(str(e))
+
     @authorized_only
     def _performance(self, update: Update, context: CallbackContext) -> None:
         """
@@ -615,6 +647,38 @@ class Telegram(RPC):
         except RPCException as e:
             self._send_msg(str(e))
 
+    @authorized_only
+    def _logs(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /logs
+        Shows the latest logs
+        """
+        try:
+            try:
+                limit = int(context.args[0])
+            except (TypeError, ValueError, IndexError):
+                limit = 10
+            logs = self._rpc_get_logs(limit)['logs']
+            msgs = ''
+            msg_template = "*{}* {}: {} \\- `{}`"
+            for logrec in logs:
+                msg = msg_template.format(escape_markdown(logrec[0], version=2),
+                                          escape_markdown(logrec[2], version=2),
+                                          escape_markdown(logrec[3], version=2),
+                                          escape_markdown(logrec[4], version=2))
+                if len(msgs + msg) + 10 >= MAX_TELEGRAM_MESSAGE_LENGTH:
+                    # Send message immediately if it would become too long
+                    self._send_msg(msgs, parse_mode=ParseMode.MARKDOWN_V2)
+                    msgs = msg + '\n'
+                else:
+                    # Append message to messages to send
+                    msgs += msg + '\n'
+
+            if msgs:
+                self._send_msg(msgs, parse_mode=ParseMode.MARKDOWN_V2)
+        except RPCException as e:
+            self._send_msg(str(e))
+
     @authorized_only
     def _edge(self, update: Update, context: CallbackContext) -> None:
         """
@@ -651,6 +715,7 @@ class Telegram(RPC):
                    "*/forcesell <trade_id>|all:* `Instantly sells the given trade or all trades, "
                    "regardless of profit`\n"
                    f"{forcebuy_text if self._config.get('forcebuy_enable', False) else ''}"
+                   "*/delete <trade_id>:* `Instantly delete the given trade in the database`\n"
                    "*/performance:* `Show performance of each finished trade grouped by pair`\n"
                    "*/daily <n>:* `Shows profit or loss per day, over the last n days`\n"
                    "*/count:* `Show number of trades running compared to allowed number of trades`"
@@ -659,6 +724,7 @@ class Telegram(RPC):
                    "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n"
                    "*/reload_config:* `Reload configuration file` \n"
                    "*/show_config:* `Show running configuration` \n"
+                   "*/logs [limit]:* `Show latest logs - defaults to 10` \n"
                    "*/whitelist:* `Show current whitelist` \n"
                    "*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs "
                    "to the blacklist.` \n"
@@ -714,7 +780,8 @@ class Telegram(RPC):
             f"*Current state:* `{val['state']}`"
         )
 
-    def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None:
+    def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN,
+                  disable_notification: bool = False) -> None:
         """
         Send given markdown message
         :param msg: message
@@ -735,7 +802,8 @@ class Telegram(RPC):
                     self._config['telegram']['chat_id'],
                     text=msg,
                     parse_mode=parse_mode,
-                    reply_markup=reply_markup
+                    reply_markup=reply_markup,
+                    disable_notification=disable_notification,
                 )
             except NetworkError as network_err:
                 # Sometimes the telegram server resets the current connection,
@@ -748,7 +816,8 @@ class Telegram(RPC):
                     self._config['telegram']['chat_id'],
                     text=msg,
                     parse_mode=parse_mode,
-                    reply_markup=reply_markup
+                    reply_markup=reply_markup,
+                    disable_notification=disable_notification,
                 )
         except TelegramError as telegram_err:
             logger.warning(

freqtrade/rpc/webhook.py

@@ -48,13 +48,13 @@ class Webhook(RPC):
             elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
                 valuedict = self._config['webhook'].get('webhooksellcancel', None)
             elif msg['type'] in (RPCMessageType.STATUS_NOTIFICATION,
-                                 RPCMessageType.CUSTOM_NOTIFICATION,
+                                 RPCMessageType.STARTUP_NOTIFICATION,
                                  RPCMessageType.WARNING_NOTIFICATION):
                 valuedict = self._config['webhook'].get('webhookstatus', None)
             else:
                 raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
             if not valuedict:
-                logger.info("Message type %s not configured for webhooks", msg['type'])
+                logger.info("Message type '%s' not configured for webhooks", msg['type'])
                 return
 
             payload = {key: value.format(**msg) for (key, value) in valuedict.items()}

freqtrade/strategy/__init__.py

@@ -1 +1,5 @@
-from freqtrade.strategy.interface import IStrategy  # noqa: F401
+# flake8: noqa: F401
+from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_prev_date,
+                                timeframe_to_seconds, timeframe_to_next_date, timeframe_to_msecs)
+from freqtrade.strategy.interface import IStrategy
+from freqtrade.strategy.strategy_helper import merge_informative_pair

freqtrade/strategy/interface.py

@@ -14,8 +14,9 @@ from pandas import DataFrame
 
 from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.exceptions import StrategyError, OperationalException
+from freqtrade.exceptions import OperationalException, StrategyError
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.exchange.exchange import timeframe_to_next_date
 from freqtrade.persistence import Trade
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.wallets import Wallets
@@ -44,6 +45,10 @@ class SellType(Enum):
     EMERGENCY_SELL = "emergency_sell"
     NONE = ""
 
+    def __str__(self):
+        # explicitly convert to String to help with exporting data.
+        return self.value
+
 
 class SellCheckTuple(NamedTuple):
     """
@@ -293,13 +298,25 @@ class IStrategy(ABC):
         if pair in self._pair_locked_until:
             del self._pair_locked_until[pair]
 
-    def is_pair_locked(self, pair: str) -> bool:
+    def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool:
         """
         Checks if a pair is currently locked
+        The 2nd, optional parameter ensures that locks are applied until the new candle arrives,
+        and not stop at 14:00:00 - while the next candle arrives at 14:00:02 leaving a gap
+        of 2 seconds for a buy to happen on an old signal.
+        :param: pair: "Pair to check"
+        :param candle_date: Date of the last candle. Optional, defaults to current date
+        :returns: locking state of the pair in question.
         """
         if pair not in self._pair_locked_until:
             return False
+        if not candle_date:
             return self._pair_locked_until[pair] >= datetime.now(timezone.utc)
+        else:
+            # Locking should happen until a new candle arrives
+            lock_time = timeframe_to_next_date(self.timeframe, candle_date)
+            # lock_time = candle_date + timedelta(minutes=timeframe_to_minutes(self.timeframe))
+            return self._pair_locked_until[pair] > lock_time
 
     def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
@@ -430,7 +447,7 @@ class IStrategy(ABC):
         if latest_date < (arrow.utcnow().shift(minutes=-(timeframe_minutes * 2 + offset))):
             logger.warning(
                 'Outdated history for pair %s. Last tick is %s minutes old',
-                pair, (arrow.utcnow() - latest_date).seconds // 60
+                pair, int((arrow.utcnow() - latest_date).total_seconds() // 60)
             )
             return False, False
 

freqtrade/strategy/strategy_helper.py

@@ -0,0 +1,48 @@
+import pandas as pd
+from freqtrade.exchange import timeframe_to_minutes
+
+
+def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
+                           timeframe: str, timeframe_inf: str, ffill: bool = True) -> pd.DataFrame:
+    """
+    Correctly merge informative samples to the original dataframe, avoiding lookahead bias.
+
+    Since dates are candle open dates, merging a 15m candle that starts at 15:00, and a
+    1h candle that starts at 15:00 will result in all candles to know the close at 16:00
+    which they should not know.
+
+    Moves the date of the informative pair by 1 time interval forward.
+    This way, the 14:00 1h candle is merged to 15:00 15m candle, since the 14:00 1h candle is the
+    last candle that's closed at 15:00, 15:15, 15:30 or 15:45.
+
+    Assuming inf_tf = '1d' - then the resulting columns will be:
+    date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d
+
+    :param dataframe: Original dataframe
+    :param informative: Informative pair, most likely loaded via dp.get_pair_dataframe
+    :param timeframe: Timeframe of the original pair sample.
+    :param timeframe_inf: Timeframe of the informative pair sample.
+    :param ffill: Forwardfill missing values - optional but usually required
+    """
+
+    minutes_inf = timeframe_to_minutes(timeframe_inf)
+    minutes = timeframe_to_minutes(timeframe)
+    if minutes >= minutes_inf:
+        # No need to forwardshift if the timeframes are identical
+        informative['date_merge'] = informative["date"]
+    else:
+        informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes_inf, 'm')
+
+    # Rename columns to be unique
+    informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns]
+
+    # Combine the 2 dataframes
+    # all indicators on the informative sample MUST be calculated before this point
+    dataframe = pd.merge(dataframe, informative, left_on='date',
+                         right_on=f'date_merge_{timeframe_inf}', how='left')
+    dataframe = dataframe.drop(f'date_merge_{timeframe_inf}', axis=1)
+
+    if ffill:
+        dataframe = dataframe.ffill()
+
+    return dataframe

freqtrade/templates/strategy_analysis_example.ipynb

@@ -34,7 +34,7 @@
     "# config = Configuration.from_files([\"config.json\"])\n",
     "\n",
     "# Define some constants\n",
-    "config[\"ticker_interval\"] = \"5m\"\n",
+    "config[\"timeframe\"] = \"5m\"\n",
     "# Name of the strategy class\n",
     "config[\"strategy\"] = \"SampleStrategy\"\n",
     "# Location of the data\n",
@@ -53,7 +53,7 @@
     "from freqtrade.data.history import load_pair_history\n",
     "\n",
     "candles = load_pair_history(datadir=data_location,\n",
-    "                            timeframe=config[\"ticker_interval\"],\n",
+    "                            timeframe=config[\"timeframe\"],\n",
     "                            pair=pair)\n",
     "\n",
     "# Confirm success\n",
@@ -136,10 +136,51 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from freqtrade.data.btanalysis import load_backtest_data\n",
+    "from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats\n",
     "\n",
-    "# Load backtest results\n",
-    "trades = load_backtest_data(config[\"user_data_dir\"] / \"backtest_results/backtest-result.json\")\n",
+    "# if backtest_dir points to a directory, it'll automatically load the last backtest file.\n",
+    "backtest_dir = config[\"user_data_dir\"] / \"backtest_results\"\n",
+    "# backtest_dir can also point to a specific file \n",
+    "# backtest_dir = config[\"user_data_dir\"] / \"backtest_results/backtest-result-2020-07-01_20-04-22.json\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# You can get the full backtest statistics by using the following command.\n",
+    "# This contains all information used to generate the backtest result.\n",
+    "stats = load_backtest_stats(backtest_dir)\n",
+    "\n",
+    "strategy = 'SampleStrategy'\n",
+    "# All statistics are available per strategy, so if `--strategy-list` was used during backtest, this will be reflected here as well.\n",
+    "# Example usages:\n",
+    "print(stats['strategy'][strategy]['results_per_pair'])\n",
+    "# Get pairlist used for this backtest\n",
+    "print(stats['strategy'][strategy]['pairlist'])\n",
+    "# Get market change (average change of all pairs from start to end of the backtest period)\n",
+    "print(stats['strategy'][strategy]['market_change'])\n",
+    "# Maximum drawdown ()\n",
+    "print(stats['strategy'][strategy]['max_drawdown'])\n",
+    "# Maximum drawdown start and end\n",
+    "print(stats['strategy'][strategy]['drawdown_start'])\n",
+    "print(stats['strategy'][strategy]['drawdown_end'])\n",
+    "\n",
+    "\n",
+    "# Get strategy comparison (only relevant if multiple strategies were compared)\n",
+    "print(stats['strategy_comparison'])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load backtested trades as dataframe\n",
+    "trades = load_backtest_data(backtest_dir)\n",
     "\n",
     "# Show value-counts per pair\n",
     "trades.groupby(\"pair\")[\"sell_reason\"].value_counts()"