Merge pull request #3731 from freqtrade/release_2020.8

Release 2020.8

.dependabot/config.yml

@@ -1,17 +0,0 @@
-version: 1
-
-update_configs:
-  - package_manager: "python"
-    directory: "/"
-    update_schedule: "weekly"
-    allowed_updates:
-    - match:
-        update_type: "all"
-    target_branch: "develop"
-
-  - package_manager: "docker"
-    directory: "/"
-    update_schedule: "daily"
-    allowed_updates:
-    - match:
-        update_type: "all"

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+- package-ecosystem: docker
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10
+  target-branch: develop

.github/workflows/ci.yml

@@ -88,7 +88,7 @@ jobs:
       run: |
         cp config.json.example config.json
         freqtrade create-userdir --userdir user_data
-        freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt
+        freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --print-all
 
     - name: Flake8
       run: |
@@ -150,7 +150,7 @@ jobs:
       run: |
         cp config.json.example config.json
         freqtrade create-userdir --userdir user_data
-        freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt
+        freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --print-all
 
     - name: Flake8
       run: |

Dockerfile

@@ -1,7 +1,7 @@
-FROM python:3.8.3-slim-buster
+FROM python:3.8.5-slim-buster
 
 RUN apt-get update \
-    && apt-get -y install curl build-essential libssl-dev \
+    && apt-get -y install curl build-essential libssl-dev sqlite3 \
     && apt-get clean \
     && pip install --upgrade pip
 

Dockerfile.armhf

@@ -1,7 +1,7 @@
 FROM --platform=linux/arm/v7 python:3.7.7-slim-buster
 
 RUN apt-get update \
-  && apt-get -y install curl build-essential libssl-dev libatlas3-base libgfortran5 \
+  && apt-get -y install curl build-essential libssl-dev libffi-dev libatlas3-base libgfortran5 sqlite3 \
   && apt-get clean \
   && pip install --upgrade pip \
   && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf

README.md

@@ -123,7 +123,6 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor
 - `/help`: Show help message
 - `/version`: Show version
 
-
 ## Development branches
 
 The project is currently setup in two main branches:

config_full.json.example

@@ -66,7 +66,7 @@
         },
         {"method": "AgeFilter", "min_days_listed": 10},
         {"method": "PrecisionFilter"},
-        {"method": "PriceFilter", "low_price_ratio": 0.01},
+        {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010},
         {"method": "SpreadFilter", "max_spread_ratio": 0.005}
     ],
     "exchange": {

docs/backtesting.md

@@ -66,7 +66,7 @@ Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies
 #### Exporting trades to file
 
 ```bash
-freqtrade backtesting --export trades
+freqtrade backtesting --export trades --config config.json --strategy SampleStrategy
 ```
 
 The exported trades can be used for [further analysis](#further-backtest-result-analysis), or can be used by the plotting script `plot_dataframe.py` in the scripts directory.
@@ -157,17 +157,32 @@ A backtesting result will look like that:
 | ADA/BTC  |      1 |           0.89 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |     1 |      0 |       0 |
 | LTC/BTC  |      1 |           0.68 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |     1 |      0 |       0 |
 | TOTAL    |      2 |           0.78 |           1.57 |       0.00007855 |           0.78 | 4:00:00        |     2 |      0 |       0 |
+=============== SUMMARY METRICS ===============
+| Metric                | Value               |
+|-----------------------+---------------------|
+| Backtesting from      | 2019-01-01 00:00:00 |
+| Backtesting to        | 2019-05-01 00:00:00 |
+| Total trades          | 429                 |
+| First trade           | 2019-01-01 18:30:00 |
+| First trade Pair      | EOS/USDT            |
+| Total Profit %        | 152.41%             |
+| Trades per day        | 3.575               |
+| Best day              | 25.27%              |
+| Worst day             | -30.67%             |
+| Avg. Duration Winners | 4:23:00             |
+| Avg. Duration Loser   | 6:55:00             |
+|                       |                     |
+| Max Drawdown          | 50.63%              |
+| Drawdown Start        | 2019-02-15 14:10:00 |
+| Drawdown End          | 2019-04-11 18:15:00 |
+| Market change         | -5.88%              |
+===============================================
 ```
 
+### Backtesting report table
+
 The 1st table contains all trades the bot made, including "left open trades".
 
-The 2nd table contains a recap of sell reasons.
-This table can tell you which area needs some additional work (i.e. all `sell_signal` trades are losses, so we should disable the sell-signal or work on improving that).
-
-The 3rd table contains all trades the bot had to `forcesell` at the end of the backtest period to present a full picture.
-This is necessary to simulate realistic behaviour, since the backtest period has to end at some point, while realistically, you could leave the bot running forever.
-These trades are also included in the first table, but are extracted separately for clarity.
-
 The last line will give you the overall performance of your strategy,
 here:
 
@@ -196,6 +211,58 @@ On the other hand, if you set a too high `minimal_roi` like `"0":  0.55`
 (55%), there is almost no chance that the bot will ever reach this profit.
 Hence, keep in mind that your performance is an integral mix of all different elements of the strategy, your configuration, and the crypto-currency pairs you have set up.
 
+### Sell reasons table
+
+The 2nd table contains a recap of sell reasons.
+This table can tell you which area needs some additional work (e.g. all or many of the `sell_signal` trades are losses, so you should work on improving the sell signal, or consider disabling it).
+
+### Left open trades table
+
+The 3rd table contains all trades the bot had to `forcesell` at the end of the backtesting period to present you the full picture.
+This is necessary to simulate realistic behavior, since the backtest period has to end at some point, while realistically, you could leave the bot running forever.
+These trades are also included in the first table, but are also shown separately in this table for clarity.
+
+### Summary metrics
+
+The last element of the backtest report is the summary metrics table.
+It contains some useful key metrics about performance of your strategy on backtesting data.
+
+```
+=============== SUMMARY METRICS ===============
+| Metric                | Value               |
+|-----------------------+---------------------|
+| Backtesting from      | 2019-01-01 00:00:00 |
+| Backtesting to        | 2019-05-01 00:00:00 |
+| Total trades          | 429                 |
+| First trade           | 2019-01-01 18:30:00 |
+| First trade Pair      | EOS/USDT            |
+| Total Profit %        | 152.41%             |
+| Trades per day        | 3.575               |
+| Best day              | 25.27%              |
+| Worst day             | -30.67%             |
+| Avg. Duration Winners | 4:23:00             |
+| Avg. Duration Loser   | 6:55:00             |
+|                       |                     |
+| Max Drawdown          | 50.63%              |
+| Drawdown Start        | 2019-02-15 14:10:00 |
+| Drawdown End          | 2019-04-11 18:15:00 |
+| Market change         | -5.88%              |
+===============================================
+
+```
+
+- `Total trades`: Identical to the total trades of the backtest output table.
+- `First trade`: First trade entered.
+- `First trade pair`: Which pair was part of the first trade.
+- `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option).
+- `Total Profit %`: Total profit per stake amount. Aligned to the TOTAL column of the first table.
+- `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy).
+- `Best day` / `Worst day`: Best and worst day based on daily profit.
+- `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades.
+- `Max Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced).
+- `Drawdown Start` / `Drawdown End`: Start and end datetimes for this largest drawdown (can also be visualized via the `plot-dataframe` sub-command).
+- `Market change`: Change of the market during the backtest period. Calculated as average of all pairs changes from the first to the last candle using the "close" column.
+
 ### Assumptions made by backtesting
 
 Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions:

docs/bot-basics.md

@@ -0,0 +1,58 @@
+# Freqtrade basics
+
+This page provides you some basic concepts on how Freqtrade works and operates.
+
+## Freqtrade terminology
+
+* Trade: Open position.
+* Open Order: Order which is currently placed on the exchange, and is not yet complete.
+* Pair: Tradable pair, usually in the format of Quote/Base (e.g. XRP/USDT).
+* Timeframe: Candle length to use (e.g. `"5m"`, `"1h"`, ...).
+* Indicators: Technical indicators (SMA, EMA, RSI, ...).
+* Limit order: Limit orders which execute at the defined limit price or better.
+* Market order: Guaranteed to fill, may move price depending on the order size.
+
+## Fee handling
+
+All profit calculations of Freqtrade include fees. For Backtesting / Hyperopt / Dry-run modes, the exchange default fee is used (lowest tier on the exchange). For live operations, fees are used as applied by the exchange (this includes BNB rebates etc.).
+
+## Bot execution logic
+
+Starting freqtrade in dry-run or live mode (using `freqtrade trade`) will start the bot and start the bot iteration loop.
+By default, loop runs every few seconds (`internals.process_throttle_secs`) and does roughly the following in the following sequence:
+
+* Fetch open trades from persistence.
+* Calculate current list of tradable pairs.
+* Download ohlcv data for the pairlist including all [informative pairs](strategy-customization.md#get-data-for-non-tradeable-pairs)  
+  This step is only executed once per Candle to avoid unnecessary network traffic.
+* Call `bot_loop_start()` strategy callback.
+* Analyze strategy per pair.
+  * Call `populate_indicators()`
+  * Call `populate_buy_trend()`
+  * Call `populate_sell_trend()`
+* Check timeouts for open orders.
+  * Calls `check_buy_timeout()` strategy callback for open buy orders.
+  * Calls `check_sell_timeout()` strategy callback for open sell orders.
+* Verifies existing positions and eventually places sell orders.
+  * Considers stoploss, ROI and sell-signal.
+  * Determine sell-price based on `ask_strategy` configuration setting.
+  * Before a sell order is placed, `confirm_trade_exit()` strategy callback is called.
+* Check if trade-slots are still available (if `max_open_trades` is reached).
+* Verifies buy signal trying to enter new positions.
+  * Determine buy-price based on `bid_strategy` configuration setting.
+  * Before a buy order is placed, `confirm_trade_entry()` strategy callback is called.
+
+This loop will be repeated again and again until the bot is stopped.
+
+## Backtesting / Hyperopt execution logic
+
+[backtesting](backtesting.md) or [hyperopt](hyperopt.md) do only part of the above logic, since most of the trading operations are fully simulated.
+
+* Load historic data for configured pairlist.
+* Calculate indicators (calls `populate_indicators()`).
+* Calls `populate_buy_trend()` and `populate_sell_trend()`
+* Loops per candle simulating entry and exit points.
+* Generate backtest report output
+
+!!! Note
+    Both Backtesting and Hyperopt include exchange default Fees in the calculation. Custom fees can be passed to backtesting / hyperopt by specifying the `--fee` argument.

docs/configuration.md

@@ -55,9 +55,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*  <br> **Datatype:** Boolean
 | `minimal_roi` | **Required.** Set the threshold as ratio the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Dict
 | `stoploss` |  **Required.** Value as ratio of the stoploss used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).  <br> **Datatype:** Float (as ratio)
-| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Boolean
+| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md#trailing-stop-loss). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Boolean
-| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Float
+| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-custom-positive-loss). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Float
-| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0` (no offset).* <br> **Datatype:** Float
+| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0` (no offset).* <br> **Datatype:** Float
 | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*  <br> **Datatype:** Boolean
 | `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
 | `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
@@ -275,27 +275,16 @@ the static list of pairs) if we should buy.
 The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`, `emergencysell`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds.
 
 This allows to buy using limit orders, sell using
-limit-orders, and create stoplosses using using market orders. It also allows to set the
+limit-orders, and create stoplosses using market orders. It also allows to set the
 stoploss "on exchange" which means stoploss order would be placed immediately once
 the buy order is fulfilled.
-If `stoploss_on_exchange` and `trailing_stop` are both set, then the bot will use `stoploss_on_exchange_interval` to check and update the stoploss on exchange periodically.
+    
-`order_types` can be set in the configuration file or in the strategy.
 `order_types` set in the configuration file overwrites values set in the strategy as a whole, so you need to configure the whole `order_types` dictionary in one place.
 
 If this is configured, the following 4 values (`buy`, `sell`, `stoploss` and
 `stoploss_on_exchange`) need to be present, otherwise the bot will fail to start.
 
-`emergencysell` is an optional value, which defaults to `market` and is used when creating stoploss on exchange orders fails.
+For information on (`emergencysell`,`stoploss_on_exchange`,`stoploss_on_exchange_interval`,`stoploss_on_exchange_limit_ratio`) please see stop loss documentation [stop loss on exchange](stoploss.md)
-The below is the default which is used if this is not configured in either strategy or configuration file.
-
-Not all Exchanges support `stoploss_on_exchange`. If an exchange supports both limit and market stoploss orders, then the value of `stoploss` will be used to determine the stoploss type.
-
-If `stoploss_on_exchange` uses limit orders, the exchange needs 2 prices, the stoploss_price and the Limit price.
-`stoploss` defines the stop-price - and limit should be slightly below this.
-
-This defaults to 0.99 / 1% (configurable via `stoploss_on_exchange_limit_ratio`).
-Calculation example: we bought the asset at 100$.
-Stop-price is 95$, then limit would be `95 * 0.99 = 94.05$` - so the stoploss will happen between 95$ and 94.05$.
 
 Syntax for Strategy:
 
@@ -662,16 +651,29 @@ Filters low-value coins which would not allow setting stoplosses.
 
 #### PriceFilter
 
-The `PriceFilter` allows filtering of pairs by price.
+The `PriceFilter` allows filtering of pairs by price. Currently the following price filters are supported:
 
-Currently, only `low_price_ratio` setting is implemented, where a raise of 1 price unit (pip) is below the `low_price_ratio` ratio.
+* `min_price`
-This option is disabled by default, and will only apply if set to <> 0.
+* `max_price`
+* `low_price_ratio`
+
+The `min_price` setting removes pairs where the price is below the specified price. This is useful if you wish to avoid trading very low-priced pairs.
+This option is disabled by default, and will only apply if set to > 0.
+
+The `max_price` setting removes pairs where the price is above the specified price. This is useful if you wish to trade only low-priced pairs.
+This option is disabled by default, and will only apply if set to > 0.
+
+The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio.
+This option is disabled by default, and will only apply if set to > 0.
+
+For `PriceFiler` at least one of its `min_price`, `max_price` or `low_price_ratio` settings must be applied.
 
 Calculation example:
 
-Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.00000012 - which is almost 10% higher than the previous value.
+Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 - one price step above would be 0.00000012, which is ~9% higher than the previous price value. You may filter out this pair by using PriceFilter with `low_price_ratio` set to 0.09 (9%) or with `min_price` set to 0.00000011, correspondingly.
 
-These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses. Here is what the PriceFilters takes over.
+!!! Warning "Low priced pairs"
+    Low priced pairs with high "1 pip movements" are dangerous since they are often illiquid and it may also be impossible to place the desired stoploss, which can often result in high losses since price needs to be rounded to the next tradable price - so instead of having a stoploss of -5%, you could end up with a stoploss of -9% simply due to price rounding.
 
 #### ShuffleFilter
 

docs/data-download.md

@@ -158,6 +158,58 @@ It'll also remove original jsongz data files (`--erase` parameter).
 freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase
 ```
 
+### Subcommand list-data
+
+You can get a list of downloaded data using the `list-data` subcommand.
+
+```
+usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
+                           [--userdir PATH] [--exchange EXCHANGE]
+                           [--data-format-ohlcv {json,jsongz}]
+                           [-p PAIRS [PAIRS ...]]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
+                        config is provided.
+  --data-format-ohlcv {json,jsongz}
+                        Storage format for downloaded candle (OHLCV) data.
+                        (default: `json`).
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Show profits for only these pairs. Pairs are space-
+                        separated.
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+```
+
+#### Example list-data
+
+```bash
+> freqtrade list-data --userdir ~/.freqtrade/user_data/
+
+Found 33 pair / timeframe combinations.
+pairs       timeframe
+----------  -----------------------------------------
+ADA/BTC     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
+ADA/ETH     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
+ETH/BTC     5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d
+ETH/USDT    5m, 15m, 30m, 1h, 2h, 4h
+```
+
 ### Pairs file
 
 In alternative to the whitelist from `config.json`, a `pairs.json` file can be used.

docs/deprecated.md

@@ -9,21 +9,20 @@ and are no longer supported. Please avoid their usage in your configuration.
 ### the `--refresh-pairs-cached` command line option
 
 `--refresh-pairs-cached` in the context of backtesting, hyperopt and edge allows to refresh candle data for backtesting.
-Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out 
+Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out as a separate freqtrade sub-command `freqtrade download-data`.
-as a seperate freqtrade subcommand `freqtrade download-data`.
 
-This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9 (master branch).
+This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9.
 
 ### The **--dynamic-whitelist** command line option
 
 This command line option was deprecated in 2018 and removed freqtrade 2019.6-dev (develop branch)
-and in freqtrade 2019.7 (master branch).
+and in freqtrade 2019.7.
 
 ### the `--live` command line option
 
 `--live` in the context of backtesting allowed to download the latest tick data for backtesting.
 Did only download the latest 500 candles, so was ineffective in getting good backtest data.
-Removed in 2019-7-dev (develop branch) and in freqtrade 2019-8 (master branch)
+Removed in 2019-7-dev (develop branch) and in freqtrade 2019.8.
 
 ### Allow running multiple pairlists in sequence
 
@@ -31,6 +30,6 @@ The former `"pairlist"` section in the configuration has been removed, and is re
 
 The old section of configuration parameters (`"pairlist"`) has been deprecated in 2019.11 and has been removed in 2020.4.
 
-### deprecation of bidVolume and askVolume from volumepairlist
+### deprecation of bidVolume and askVolume from volume-pairlist
 
 Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4.

docs/developer.md

@@ -85,6 +85,35 @@ docker-compose exec freqtrade_develop /bin/bash
 
 ![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png)
 
+## ErrorHandling
+
+Freqtrade Exceptions all inherit from `FreqtradeException`.
+This general class of error should however not be used directly. Instead, multiple specialized sub-Exceptions exist.
+
+Below is an outline of exception inheritance hierarchy:
+
+```
++ FreqtradeException
+|
++---+ OperationalException
+|
++---+ DependencyException
+|   |
+|   +---+ PricingError
+|   |
+|   +---+ ExchangeError
+|       |
+|       +---+ TemporaryError
+|       |
+|       +---+ DDosProtection
+|       |
+|       +---+ InvalidOrderException
+|           |
+|           +---+ RetryableOrderError
+|
++---+ StrategyError
+```
+
 ## Modules
 
 ### Dynamic Pairlist

docs/edge.md

@@ -6,7 +6,8 @@ This page explains how to use Edge Positioning module in your bot in order to en
     Edge positioning is not compatible with dynamic (volume-based) whitelist.
 
 !!! Note
-    Edge does not consider anything else than buy/sell/stoploss signals. So trailing stoploss, ROI, and everything else are ignored in its calculation.
+    Edge does not consider anything other than *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file.
+    Therefore, it is important to understand that Edge can improve the performance of some trading strategies but *decrease* the performance of others.
 
 ## Introduction
 
@@ -89,7 +90,7 @@ You can also use this value to evaluate the effectiveness of modifications to th
 
 ## How does it work?
 
-If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over *N* trades for each stoploss. Here is an example:
+Edge combines dynamic stoploss, dynamic positions, and whitelist generation into one isolated module which is then applied to the trading strategy. If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over *N* trades for each stoploss. Here is an example:
 
 | Pair   |      Stoploss      |  Win Rate | Risk Reward Ratio | Expectancy |
 |----------|:-------------:|-------------:|------------------:|-----------:|
@@ -186,6 +187,12 @@ An example of its output:
 | APPC/BTC  |      -0.02 |       0.44 |                2.28 |                   1.27 |         0.44 |                       25 |                       43 |
 | NEBL/BTC  |      -0.03 |       0.63 |                1.29 |                   0.58 |         0.44 |                       19 |                       59 |
 
+Edge produced the above table by comparing `calculate_since_number_of_days` to `minimum_expectancy` to find `min_trade_number` historical information based on the config file. The timerange Edge uses for its comparisons can be further limited by using the `--timerange` switch.
+
+In live and dry-run modes, after the `process_throttle_secs` has passed, Edge will again process `calculate_since_number_of_days` against `minimum_expectancy` to find `min_trade_number`. If no `min_trade_number` is found, the bot will return "whitelist empty". Depending on the trade strategy being deployed, "whitelist empty" may be return much of the time - or *all* of the time. The use of Edge may also cause trading to occur in bursts, though this is rare.
+
+If you encounter "whitelist empty" a lot, condsider tuning `calculate_since_number_of_days`, `minimum_expectancy`  and `min_trade_number` to align to the trading frequency of your strategy.
+
 ### Update cached pairs with the latest data
 
 Edge requires historic data the same way as backtesting does.

docs/faq.md

@@ -1,5 +1,9 @@
 # Freqtrade FAQ
 
+## Beginner Tips & Tricks
+
+* When you work with your strategy & hyperopt file you should use a proper code editor like vscode or Pycharm. A good code editor will provide syntax highlighting as well as line numbers, making it easy to find syntax errors (most likely, pointed out by Freqtrade during startup).
+
 ## Freqtrade common issues
 
 ### The bot does not start
@@ -15,10 +19,12 @@ This could have the following reasons:
 
 ### I have waited 5 minutes, why hasn't the bot made any trades yet?!
 
-Depending on the buy strategy, the amount of whitelisted coins, the
+* Depending on the buy strategy, the amount of whitelisted coins, the
 situation of the market etc, it can take up to hours to find good entry
 position for a trade. Be patient!
 
+* Or it may because of a configuration error? Best check the logs, it's usually telling you if the bot is simply not getting buy signals (only heartbeat messages), or if there is something wrong (errors / exceptions in the log).
+
 ### I have made 12 trades already, why is my total profit negative?!
 
 I understand your disappointment but unfortunately 12 trades is just
@@ -129,25 +135,27 @@ to find a great result (unless if you are very lucky), so you probably
 have to run it for 10.000 or more. But it will take an eternity to
 compute.
 
-We recommend you to run it at least 10.000 epochs:
+Since hyperopt uses Bayesian search, running for too many epochs may not produce greater results.
+
+It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epocs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going. 
 
 ```bash
-freqtrade hyperopt -e 10000
+freqtrade hyperopt -e 1000
 ```
 
 or if you want intermediate result to see
 
 ```bash
-for i in {1..100}; do freqtrade hyperopt -e 100; done
+for i in {1..100}; do freqtrade hyperopt -e 1000; done
 ```
 
-### Why it is so long to run hyperopt?
+### Why does it take a long time to run hyperopt?
 
-Finding a great Hyperopt results takes time.
+* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
 
-If you wonder why it takes a while to find great hyperopt results
+* If you wonder why it can take from 20 minutes to days to do 1000 epocs here are some answers:
 
-This answer was written during the under the release 0.15.1, when we had:
+This answer was written during the release 0.15.1, when we had:
 
 - 8 triggers
 - 9 guards: let's say we evaluate even 10 values from each
@@ -157,7 +165,14 @@ The following calculation is still very rough and not very precise
 but it will give the idea. With only these triggers and guards there is
 already 8\*10^9\*10 evaluations. A roughly total of 80 billion evals.
 Did you run 100 000 evals? Congrats, you've done roughly 1 / 100 000 th
-of the search space.
+of the search space, assuming that the bot never tests the same parameters more than once.
+
+* The time it takes to run 1000 hyperopt epocs depends on things like: The available cpu, harddisk, ram, timeframe, timerange, indicator settings, indicator count, amount of coins that hyperopt test strategies on and the resulting trade count - which can be 650 trades in a year or 10.0000 trades depending if the strategy aims for big profits by trading rarely or for many low profit trades. 
+
+Example: 4% profit 650 times vs 0,3% profit a trade 10.000 times in a year. If we assume you set the --timerange to 365 days. 
+
+Example: 
+`freqtrade --config config.json --strategy SampleStrategy --hyperopt SampleHyperopt -e 1000 --timerange 20190601-20200601`
 
 ## Edge module
 

docs/hyperopt.md

@@ -370,6 +370,9 @@ By default, hyperopt prints colorized results -- epochs with positive profit are
 
 You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option.
 
+!!! Note "Windows and color output"
+    Windows does not support color-output nativly, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL.
+
 ### Understand Hyperopt ROI results
 
 If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table:
@@ -498,8 +501,3 @@ After you run Hyperopt for the desired amount of epochs, you can later list all
 Once the optimized strategy has been implemented into your strategy, you should backtest this strategy to make sure everything is working as expected.
 
 To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same set of arguments `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting.
-
-## Next Step
-
-Now you have a perfect bot and want to control it from Telegram. Your
-next step is to learn the [Telegram usage](telegram-usage.md).

docs/plotting.md

@@ -224,7 +224,8 @@ Possible options for the `freqtrade plot-profit` subcommand:
 
 ```
 usage: freqtrade plot-profit [-h] [-v] [--logfile FILE] [-V] [-c PATH]
-                             [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]]
+                             [-d PATH] [--userdir PATH] [-s NAME]
+                             [--strategy-path PATH] [-p PAIRS [PAIRS ...]]
                              [--timerange TIMERANGE] [--export EXPORT]
                              [--export-filename PATH] [--db-url PATH]
                              [--trade-source {DB,file}] [-i TIMEFRAME]
@@ -270,6 +271,11 @@ Common arguments:
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 
+Strategy arguments:
+  -s NAME, --strategy NAME
+                        Specify strategy class name which will be used by the
+                        bot.
+  --strategy-path PATH  Specify additional strategy lookup path.
 ```
 
 The `-p/--pairs`  argument, can be used to limit the pairs that are considered for this calculation.
@@ -279,7 +285,7 @@ Examples:
 Use custom backtest-export file
 
 ``` bash
-freqtrade plot-profit  -p LTC/BTC --export-filename user_data/backtest_results/backtest-result-Strategy005.json
+freqtrade plot-profit  -p LTC/BTC --export-filename user_data/backtest_results/backtest-result.json
 ```
 
 Use custom database

docs/requirements-docs.txt

@@ -1,2 +1,2 @@
-mkdocs-material==5.3.3
+mkdocs-material==5.5.8
 mdx_truly_sane_lists==1.2

docs/rest-api.md

@@ -46,7 +46,7 @@ secrets.token_hex()
 
 ### Configuration with docker
 
-If you run your bot using docker, you'll need to have the bot listen to incomming connections. The security is then handled by docker.
+If you run your bot using docker, you'll need to have the bot listen to incoming connections. The security is then handled by docker.
 
 ``` json
     "api_server": {
@@ -106,26 +106,30 @@ python3 scripts/rest_client.py --config rest_config.json <command> [optional par
 
 ## Available commands
 
-|  Command | Default | Description |
+|  Command | Description |
-|----------|---------|-------------|
+|----------|-------------|
-| `start` | | Starts the trader
+| `ping` | Simple command testing the API Readiness - requires no authentication.
-| `stop` | | Stops the trader
+| `start` | Starts the trader
-| `stopbuy` | | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
+| `stop` | Stops the trader
-| `reload_config` | | Reloads the configuration file
+| `stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
-| `show_config` | | Shows part of the current configuration with relevant settings to operation
+| `reload_config` | Reloads the configuration file
-| `status` | | Lists all open trades
+| `trades` | List last trades.
-| `count` | | Displays number of trades used and available
+| `delete_trade <trade_id>` | Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange.
-| `profit` | | Display a summary of your profit/loss from close trades and some stats about your performance
+| `show_config` | Shows part of the current configuration with relevant settings to operation
-| `forcesell <trade_id>` | | Instantly sells the given trade  (Ignoring `minimum_roi`).
+| `logs` | Shows last log messages
-| `forcesell all` | | Instantly sells all open trades (Ignoring `minimum_roi`).
+| `status` | Lists all open trades
-| `forcebuy <pair> [rate]` | | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True)
+| `count` | Displays number of trades used and available
-| `performance` | | Show performance of each finished trade grouped by pair
+| `profit` | Display a summary of your profit/loss from close trades and some stats about your performance
-| `balance` | | Show account balance per currency
+| `forcesell <trade_id>` | Instantly sells the given trade  (Ignoring `minimum_roi`).
-| `daily <n>` | 7 | Shows profit or loss per day, over the last n days
+| `forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`).
-| `whitelist` | | Show the current whitelist
+| `forcebuy <pair> [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True)
-| `blacklist [pair]` | | Show the current blacklist, or adds a pair to the blacklist.
+| `performance` | Show performance of each finished trade grouped by pair
-| `edge` | | Show validated pairs by Edge if it is enabled.
+| `balance` | Show account balance per currency
-| `version` | | Show version
+| `daily <n>` | Shows profit or loss per day, over the last n days (n defaults to 7)
+| `whitelist` | Show the current whitelist
+| `blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist.
+| `edge` | Show validated pairs by Edge if it is enabled.
+| `version` | Show version
 
 Possible commands can be listed from the rest-client script using the `help` command.
 
@@ -135,78 +139,83 @@ python3 scripts/rest_client.py help
 
 ``` output
 Possible commands:
+
 balance
-        Get the account balance
+	Get the account balance.
-        :returns: json object
 
 blacklist
-        Show the current blacklist
+	Show the current blacklist.
+
         :param add: List of coins to add (example: "BNB/BTC")
-        :returns: json object
 
 count
-        Returns the amount of open trades
+	Return the amount of open trades.
-        :returns: json object
 
 daily
-        Returns the amount of open trades
+	Return the amount of open trades.
-        :returns: json object
+
+delete_trade
+	Delete trade from the database.
+        Tries to close open orders. Requires manual handling of this asset on the exchange.
+
+        :param trade_id: Deletes the trade with this ID from the database.
 
 edge
-        Returns information about edge
+	Return information about edge.
-        :returns: json object
 
 forcebuy
-        Buy an asset
+	Buy an asset.
+
         :param pair: Pair to buy (ETH/BTC)
         :param price: Optional - price to buy
-        :returns: json object of the trade
 
 forcesell
-        Force-sell a trade
+	Force-sell a trade.
+
         :param tradeid: Id of the trade (can be received via status command)
-        :returns: json object
+
+logs
+	Show latest logs.
+
+        :param limit: Limits log messages to the last <limit> logs. No limit to get all the trades.
 
 performance
-        Returns the performance of the different coins
+	Return the performance of the different coins.
-        :returns: json object
 
 profit
-        Returns the profit summary
+	Return the profit summary.
-        :returns: json object
 
 reload_config
-        Reload configuration
+	Reload configuration.
-        :returns: json object
 
 show_config
+	
         Returns part of the configuration, relevant for trading operations.
-        :return: json object containing the version
 
 start
-        Start the bot if it's in stopped state.
+	Start the bot if it's in the stopped state.
-        :returns: json object
 
 status
-        Get the status of open trades
+	Get the status of open trades.
-        :returns: json object
 
 stop
-        Stop the bot. Use start to restart
+	Stop the bot. Use `start` to restart.
-        :returns: json object
 
 stopbuy
-        Stop buying (but handle sells gracefully).
+	Stop buying (but handle sells gracefully). Use `reload_config` to reset.
-        use reload_config to reset
+
-        :returns: json object
+trades
+	Return trades history.
+
+        :param limit: Limits trades to the X last trades. No limit to get all the trades.
 
 version
-        Returns the version of the bot
+	Return the version of the bot.
-        :returns: json object containing the version
 
 whitelist
-        Show the current whitelist
+	Show the current whitelist.
-        :returns: json object
+
+
 ```
 
 ## Advanced API usage using JWT tokens

docs/sandbox-testing.md

@@ -1,104 +1,59 @@
 # Sandbox API testing
 
-Where an exchange provides a sandbox for risk-free integration, or end-to-end, testing CCXT provides access to these.
+Some exchanges provide sandboxes or testbeds for risk-free testing, while running the bot against a real exchange.
+With some configuration, freqtrade (in combination with ccxt) provides access to these.
 
-This document is a *light overview of configuring Freqtrade and GDAX sandbox.
+This document is an overview to configure Freqtrade to be used with sandboxes.
-This can be useful to developers and trader alike as Freqtrade is quite customisable.
+This can be useful to developers and trader alike.
 
-When testing your API connectivity, make sure to use the following URLs.
+## Exchanges known to have a sandbox / testnet
-***Website**
+
-https://public.sandbox.gdax.com
+* [binance](https://testnet.binance.vision/)
-***REST API**
+* [coinbasepro](https://public.sandbox.pro.coinbase.com)
-https://api-public.sandbox.gdax.com
+* [gemini](https://exchange.sandbox.gemini.com/)
+* [huobipro](https://www.testnet.huobi.pro/)
+* [kucoin](https://sandbox.kucoin.com/)
+* [phemex](https://testnet.phemex.com/) 
+
+!!! Note
+    We did not test correct functioning of all of the above testnets. Please report your experiences with each sandbox.
 
 ---
 
-# Configure a Sandbox account on Gdax
+## Configure a Sandbox account
 
-Aim of this document section
+When testing your API connectivity, make sure to use the appropriate sandbox / testnet URL.
 
-- An sanbox account
+In general, you should follow these steps to enable an exchange's sandbox:
-- create 2FA (needed to create an API)
-- Add test 50BTC to account 
-- Create :
-- - API-KEY
-- - API-Secret
-- - API Password
 
-## Acccount
+* Figure out if an exchange has a sandbox (most likely by using google or the exchange's support documents)
+* Create a sandbox account (often the sandbox-account requires separate registration)
+* [Add some test assets to account](#add-test-funds)
+* Create API keys
 
-This link will redirect to the sandbox main page to login / create account dialogues:
+### Add test funds
-https://public.sandbox.pro.coinbase.com/orders/
 
-After registration and Email confimation you wil be redirected into your sanbox account.  It is easy to verify you're in sandbox by checking the URL bar.
+Usually, sandbox exchanges allow depositing funds directly via web-interface.
-> https://public.sandbox.pro.coinbase.com/
+You should make sure to have a realistic amount of funds available to your test-account, so results are representable of your real account funds.
 
-## Enable 2Fa (a prerequisite to creating sandbox API Keys)
+!!! Warning
+    Test exchanges will **NEVER** require your real credit card or banking details!
 
-From within sand box site select your profile, top right.
+## Configure freqtrade to use a exchange's sandbox
->Or as a direct link: https://public.sandbox.pro.coinbase.com/profile
 
-From the menu panel to the left of the screen select
+### Sandbox URLs
-
-> Security: "*View or Update*"
-
-In the new site select "enable authenticator" as typical google Authenticator.
-
-- open Google Authenticator on your phone
-- scan barcode
-- enter your generated 2fa
-
-## Enable API Access
-
-From within sandbox select profile>api>create api-keys
->or as a direct link: https://public.sandbox.pro.coinbase.com/profile/api
-
-Click on "create one" and ensure  **view** and **trade**  are "checked" and sumbit your 2FA
-
-- **Copy and paste the Passphase** into a notepade this will be needed later
-- **Copy and paste the API Secret** popup into a notepad this will needed later
-- **Copy and paste the API Key** into a notepad this will needed later
-
-## Add 50 BTC test funds
-
-To add funds, use the web interface deposit and withdraw buttons.
-
-To begin select 'Wallets' from the top menu.
-> Or as a direct link: https://public.sandbox.pro.coinbase.com/wallets
-
-- Deposits (bottom left of screen)
-- - Deposit Funds Bitcoin
-- - - Coinbase BTC Wallet
-- - - - Max (50 BTC)
-- - - - - Deposit
-
-*This process may be repeated for other currencies, ETH as example*
-
----
-
-# Configure Freqtrade to use Gax Sandbox 
-
-The aim of this document section
-
-- Enable sandbox URLs in Freqtrade
-- Configure API
-- - secret
-- - key
-- - passphrase
- 
-## Sandbox URLs
 
 Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade.
 These include `['test']` and `['api']`.
 
-- `[Test]` if available will point to an Exchanges sandbox. 
+* `[Test]` if available will point to an Exchanges sandbox.
-- `[Api]` normally used, and resolves to live API target on the exchange 
+* `[Api]` normally used, and resolves to live API target on the exchange.
 
 To make use of sandbox / test add "sandbox": true, to your config.json
 
 ```json
   "exchange": {
-        "name": "gdax",
+        "name": "coinbasepro",
         "sandbox": true,
         "key": "5wowfxemogxeowo;heiohgmd",
         "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==",
@@ -106,36 +61,57 @@ To make use of sandbox / test add "sandbox": true, to your config.json
         "outdated_offset": 5
         "pair_whitelist": [
             "BTC/USD"
+        ]
+  },
+  "datadir": "user_data/data/coinbasepro_sandbox"
 ```
 
-Also insert your 
+Also the following information:
 
-- api-key  (noted earlier)
+* api-key (created for the sandbox webpage)
-- api-secret (noted earlier)
+* api-secret (noted earlier)
-- password (the passphrase - noted earlier)
+* password (the passphrase - noted earlier)
+
+!!! Tip "Different data directory"
+    We also recommend to set `datadir` to something identifying downloaded data as sandbox data, to avoid having sandbox data mixed with data from the real exchange.
+    This can be done by adding the `"datadir"` key to the configuration.
+    Now, whenever you use this configuration, your data directory will be set to this directory.
 
 ---
 
 ## You should now be ready to test your sandbox
 
-Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox.
+Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox. Also make sure to select a pair which shows at least some decent value (which very often is BTC/<somestablecoin>).
-** Typically the  BTC/USD has the most activity in sandbox to test against.
 
-## GDAX - Old Candles problem
+## Common problems with sandbox exchanges
 
-It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks.
+Sandbox exchange instances often have very low volume, which can cause some problems which usually are not seen on a real exchange instance.
 
-To disable this check, add / change the `"outdated_offset"` parameter in the exchange section of your configuration to adjust for this delay.
+### Old Candles problem
-Example based on the above configuration:
 
-```json
+Since Sandboxes often have low volume, candles can be quite old and show no volume.
-  "exchange": {
+To disable the error "Outdated history for pair ...", best increase the parameter `"outdated_offset"` to a number that seems realistic for the sandbox you're using.
-        "name": "gdax",
+
-        "sandbox": true,
+### Unfilled orders
-        "key": "5wowfxemogxeowo;heiohgmd",
+
-        "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==",
+Sandboxes often have very low volumes - which means that many trades can go unfilled, or can go unfilled for a very long time.
-        "password": "1bkjfkhfhfu6sr",
+
-        "outdated_offset": 30
+To mitigate this, you can try to match the first order on the opposite orderbook side using the following configuration:
-        "pair_whitelist": [
+
-            "BTC/USD"
+``` jsonc
-```
+  "order_types": {
+    "buy": "limit",
+    "sell": "limit"
+    // ...
+  },
+  "bid_strategy": {
+    "price_side": "ask",
+    // ...
+  },
+  "ask_strategy":{
+    "price_side": "bid",
+    // ...
+  },
+  ```
+
+  The configuration is similar to the suggested configuration for market orders - however by using limit-orders you can avoid moving the price too much, and you can set the worst price you might get.

docs/sql_cheatsheet.md

@@ -13,6 +13,15 @@ Feel free to use a visual Database editor like SqliteBrowser if you feel more co
 sudo apt-get install sqlite3
 ```
 
+### Using sqlite3 via docker-compose
+
+The freqtrade docker image does contain sqlite3, so you can edit the database without having to install anything on the host system.
+
+``` bash
+docker-compose exec freqtrade /bin/bash
+sqlite3 <databasefile>.sqlite
+```
+
 ## Open the DB
 
 ```bash
@@ -100,8 +109,8 @@ UPDATE trades
 SET is_open=0,
   close_date=<close_date>,
   close_rate=<close_rate>,
-  close_profit=close_rate/open_rate-1,
+  close_profit = close_rate / open_rate - 1,
-  close_profit_abs = (amount * <close_rate> * (1 - fee_close) - (amount * open_rate * 1 - fee_open)),
+  close_profit_abs = (amount * <close_rate> * (1 - fee_close) - (amount * (open_rate * (1 - fee_open)))),
   sell_reason=<sell_reason>
 WHERE id=<trade_ID_to_update>;
 ```
@@ -111,24 +120,39 @@ WHERE id=<trade_ID_to_update>;
 ```sql
 UPDATE trades
 SET is_open=0,
-  close_date='2017-12-20 03:08:45.103418',
+  close_date='2020-06-20 03:08:45.103418',
   close_rate=0.19638016,
   close_profit=0.0496,
-  close_profit_abs = (amount * 0.19638016 * (1 - fee_close) - (amount * open_rate * 1 - fee_open))
+  close_profit_abs = (amount * 0.19638016 * (1 - fee_close) - (amount * (open_rate * (1 - fee_open)))),
   sell_reason='force_sell'  
 WHERE id=31;
 ```
 
-## Insert manually a new trade
+## Manually insert a new trade
 
 ```sql
 INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date)
-VALUES ('bittrex', 'ETH/BTC', 1, 0.0025, 0.0025, <open_rate>, <stake_amount>, <amount>, '<datetime>')
+VALUES ('binance', 'ETH/BTC', 1, 0.0025, 0.0025, <open_rate>, <stake_amount>, <amount>, '<datetime>')
 ```
 
-##### Example:
+### Insert trade example
 
 ```sql
 INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date)
-VALUES ('bittrex', 'ETH/BTC', 1, 0.0025, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000')
+VALUES ('binance', 'ETH/BTC', 1, 0.0025, 0.0025, 0.00258580, 0.002, 0.7715262081, '2020-06-28 12:44:24.000000')
 ```
+
+## Remove trade from the database
+
+Maybe you'd like to remove a trade from the database, because something went wrong.
+
+```sql
+DELETE FROM trades WHERE id = <tradeid>;
+```
+
+```sql
+DELETE FROM trades WHERE id = 31;
+```
+
+!!! Warning
+    This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause.

docs/stoploss.md

@@ -6,7 +6,63 @@ For example, value `-0.10` will cause immediate sell if the profit dips below -1
 Most of the strategy files already include the optimal `stoploss` value.
 
 !!! Info
-    All stoploss properties mentioned in this file can be set in the Strategy, or in the configuration. Configuration values will override the strategy values.
+    All stoploss properties mentioned in this file can be set in the Strategy, or in the configuration.  
+    <ins>Configuration values will override the strategy values.</ins>
+
+## Stop Loss On-Exchange/Freqtrade
+
+Those stoploss modes can be *on exchange* or *off exchange*.
+
+These modes can be configured with these values:
+
+``` python
+    'emergencysell': 'market',
+    'stoploss_on_exchange': False
+    'stoploss_on_exchange_interval': 60,
+    'stoploss_on_exchange_limit_ratio': 0.99
+```
+
+!!! Note
+    Stoploss on exchange is only supported for Binance (stop-loss-limit), Kraken (stop-loss-market) and FTX (stop limit and stop-market) as of now.  
+    <ins>Do not set too low stoploss value if using stop loss on exchange!</ins>  
+    If set to low/tight then you have greater risk of missing fill on the order and stoploss will not work
+
+### stoploss_on_exchange and stoploss_on_exchange_limit_ratio
+Enable or Disable stop loss on exchange.
+If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled.
+
+If `stoploss_on_exchange` uses limit orders, the exchange needs 2 prices, the stoploss_price and the Limit price.  
+`stoploss` defines the stop-price where the limit order is placed - and limit should be slightly below this.  
+If an exchange supports both limit and market stoploss orders, then the value of `stoploss` will be used to determine the stoploss type.  
+
+Calculation example: we bought the asset at 100$.  
+Stop-price is 95$, then limit would be `95 * 0.99 = 94.05$` - so the limit order fill can happen between 95$ and 94.05$.  
+
+For example, assuming the stoploss is on exchange, and trailing stoploss is enabled, and the market is going up, then the bot automatically cancels the previous stoploss order and puts a new one with a stop value higher than the previous stoploss order.
+
+### stoploss_on_exchange_interval
+In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary.  
+The bot cannot do these every 5 seconds (at each iteration), otherwise it would get banned by the exchange.
+So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute).
+This same logic will reapply a stoploss order on the exchange should you cancel it accidentally.
+
+### emergencysell
+`emergencysell` is an optional value, which defaults to `market` and is used when creating stop loss on exchange orders fails.
+The below is the default which is used if not changed in strategy or configuration file.
+
+Example from strategy file:
+
+``` python
+order_types = {
+    'buy': 'limit',
+    'sell': 'limit',
+    'emergencysell': 'market',
+    'stoploss': 'market',
+    'stoploss_on_exchange': True,
+    'stoploss_on_exchange_interval': 60,
+    'stoploss_on_exchange_limit_ratio': 0.99
+}
+```
 
 ## Stop Loss Types
 
@@ -17,29 +73,29 @@ At this stage the bot contains the following stoploss support modes:
 3. Trailing stop loss, custom positive loss.
 4. Trailing stop loss only once the trade has reached a certain offset.
 
-Those stoploss modes can be *on exchange* or *off exchange*. If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled.
+### Static Stop Loss
-
-In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary.
-
-For example, assuming the stoploss is on exchange, and trailing stoploss is enabled, and the market is going up, then the bot automatically cancels the previous stoploss order and puts a new one with a stop value higher than the previous stoploss order.
-The bot cannot do this every 5 seconds (at each iteration), otherwise it would get banned by the exchange.
-So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute).
-This same logic will reapply a stoploss order on the exchange should you cancel it accidentally.
-
-!!! Note
-    Stoploss on exchange is only supported for Binance (stop-loss-limit), Kraken (stop-loss-market) and FTX (stop limit and stop-market) as of now.
-
-## Static Stop Loss
 
 This is very simple, you define a stop loss of x (as a ratio of price, i.e. x * 100% of price). This will try to sell the asset once the loss exceeds the defined loss.
 
-## Trailing Stop Loss
+Example of stop loss:
+
+``` python
+    stoploss = -0.10
+```
+
+For example, simplified math:
+* the bot buys an asset at a price of 100$
+* the stop loss is defined at -10%
+* the stop loss would get triggered once the asset drops below 90$
+
+### Trailing Stop Loss
 
 The initial value for this is `stoploss`, just as you would define your static Stop loss.
 To enable trailing stoploss:
 
 ``` python
-trailing_stop = True
+    stoploss = -0.10
+    trailing_stop = True
 ```
 
 This will now activate an algorithm, which automatically moves the stop loss up every time the price of your asset increases.
@@ -47,35 +103,43 @@ This will now activate an algorithm, which automatically moves the stop loss up
 For example, simplified math:
 
 * the bot buys an asset at a price of 100$
-* the stop loss is defined at 2%
+* the stop loss is defined at -10%
-* the stop loss would get triggered once the asset dropps below 98$
+* the stop loss would get triggered once the asset drops below 90$
 * assuming the asset now increases to 102$
-* the stop loss will now be 2% of 102$ or 99.96$
+* the stop loss will now be -10% of 102$ = 91.8$
-* now the asset drops in value to 101$, the stop loss will still be 99.96$ and would trigger at 99.96$.
+* now the asset drops in value to 101$, the stop loss will still be 91.8$ and would trigger at 91.8$.
 
-In summary: The stoploss will be adjusted to be always be 2% of the highest observed price.
+In summary: The stoploss will be adjusted to be always be -10% of the highest observed price.
 
-### Custom positive stoploss
+### Trailing stop loss, custom positive loss
 
-It is also possible to have a default stop loss, when you are in the red with your buy, but once your profit surpasses a certain percentage, the system will utilize a new stop loss, which can have a different value.
+It is also possible to have a default stop loss, when you are in the red with your buy (buy - fee), but once you hit positive result the system will utilize a new stop loss, which can have a different value.
-For example your default stop loss is 5%, but once you have 1.1% profit, it will be changed to be only a 1% stop loss, which trails the green candles until it goes below them.
+For example, your default stop loss is -10%, but once you have more than 0% profit (example 0.1%) a different trailing stoploss will be used.
 
-Both values require `trailing_stop` to be set to true.
+!!! Note
+    If you want the stoploss to only be changed when you break even of making a profit (what most users want) please refer to next section with [offset enabled](#Trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset).
+
+Both values require `trailing_stop` to be set to true and `trailing_stop_positive` with a value.
 
 ``` python
-    trailing_stop_positive = 0.01
+    stoploss = -0.10
-    trailing_stop_positive_offset = 0.011
+    trailing_stop = True
+    trailing_stop_positive = 0.02
 ```
 
-The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit.
+For example, simplified math:
+
+* the bot buys an asset at a price of 100$
+* the stop loss is defined at -10%
+* the stop loss would get triggered once the asset drops below 90$
+* assuming the asset now increases to 102$
+* the stop loss will now be -2% of 102$ = 99.96$ (99.96$ stop loss will be locked in and will follow asset price increasements with -2%)
+* now the asset drops in value to 101$, the stop loss will still be 99.96$ and would trigger at 99.96$
+
+The 0.02 would translate to a -2% stop loss.
 Before this, `stoploss` is used for the trailing stoploss.
 
-Read the [next section](#trailing-only-once-offset-is-reached) to keep stoploss at 5% of the entry point.
+### Trailing stop loss only once the trade has reached a certain offset
-
-!!! Tip
-    Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade.
-
-### Trailing only once offset is reached
 
 It is also possible to use a static stoploss until the offset is reached, and then trail the trade to take profits once the market turns.
 
@@ -84,20 +148,31 @@ This option can be used with or without `trailing_stop_positive`, but uses `trai
 
 ``` python
     trailing_stop_positive_offset = 0.011
-    trailing_only_offset_is_reached = true
+    trailing_only_offset_is_reached = True
 ```
 
-Simplified example:
+Configuration (offset is buyprice + 3%):
 
 ``` python
-    stoploss = 0.05
+    stoploss = -0.10
+    trailing_stop = True
+    trailing_stop_positive = 0.02
     trailing_stop_positive_offset = 0.03
     trailing_only_offset_is_reached = True
 ```
 
+For example, simplified math:
+
 * the bot buys an asset at a price of 100$
-* the stop loss is defined at 5%
+* the stop loss is defined at -10%
-* the stop loss will remain at 95% until profit reaches +3%
+* the stop loss would get triggered once the asset drops below 90$
+* stoploss will remain at 90$ unless asset increases to or above our configured offset
+* assuming the asset now increases to 103$ (where we have the offset configured)
+* the stop loss will now be -2% of 103$ = 100.94$
+* now the asset drops in value to 101$, the stop loss will still be 100.94$ and would trigger at 100.94$
+
+!!! Tip
+    Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade.
 
 ## Changing stoploss on open trades
 

docs/strategy-advanced.md

@@ -1,7 +1,12 @@
 # Advanced Strategies
 
 This page explains some advanced concepts available for strategies.
-If you're just getting started, please be familiar with the methods described in the [Strategy Customization](strategy-customization.md) documentation first.
+If you're just getting started, please be familiar with the methods described in the [Strategy Customization](strategy-customization.md) documentation and with the [Freqtrade basics](bot-basics.md) first.
+
+[Freqtrade basics](bot-basics.md) describes in which sequence each method described below is called, which can be helpful to understand which method to use for your custom needs.
+
+!!! Note
+    All callback methods described below should only be implemented in a strategy if they are actually used.
 
 ## Custom order timeout rules
 
@@ -89,3 +94,129 @@ class Awesomestrategy(IStrategy):
             return True
         return False
 ```
+
+## Bot loop start callback
+
+A simple callback which is called once at the start of every bot throttling iteration.
+This can be used to perform calculations which are pair independent (apply to all pairs), loading of external data, etc.
+
+``` python
+import requests
+
+class Awesomestrategy(IStrategy):
+
+    # ... populate_* methods
+
+    def bot_loop_start(self, **kwargs) -> None:
+        """
+        Called at the start of the bot iteration (one loop).
+        Might be used to perform pair-independent tasks
+        (e.g. gather some remote resource for comparison)
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        """
+        if self.config['runmode'].value in ('live', 'dry_run'):
+            # Assign this to the class by using self.*
+            # can then be used by populate_* methods
+            self.remote_data = requests.get('https://some_remote_source.example.com')
+
+```
+
+## Bot order confirmation
+
+### Trade entry (buy order) confirmation
+
+`confirm_trade_entry()` can be used to abort a trade entry at the latest second (maybe because the price is not what we expect).
+
+``` python
+class Awesomestrategy(IStrategy):
+
+    # ... populate_* methods
+
+    def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float,
+                            time_in_force: str, **kwargs) -> bool:
+        """
+        Called right before placing a buy order.
+        Timing for this function is critical, so avoid doing heavy computations or
+        network requests in this method.
+
+        For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
+
+        When not implemented by a strategy, returns True (always confirming).
+
+        :param pair: Pair that's about to be bought.
+        :param order_type: Order type (as configured in order_types). usually limit or market.
+        :param amount: Amount in target (quote) currency that's going to be traded.
+        :param rate: Rate that's going to be used when using limit orders
+        :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        :return bool: When True is returned, then the buy-order is placed on the exchange.
+            False aborts the process
+        """
+        return True
+
+```
+
+### Trade exit (sell order) confirmation
+
+`confirm_trade_exit()` can be used to abort a trade exit (sell) at the latest second (maybe because the price is not what we expect).
+
+``` python
+from freqtrade.persistence import Trade
+
+
+class Awesomestrategy(IStrategy):
+
+    # ... populate_* methods
+
+    def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float,
+                           rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool:
+        """
+        Called right before placing a regular sell order.
+        Timing for this function is critical, so avoid doing heavy computations or
+        network requests in this method.
+
+        For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
+
+        When not implemented by a strategy, returns True (always confirming).
+
+        :param pair: Pair that's about to be sold.
+        :param order_type: Order type (as configured in order_types). usually limit or market.
+        :param amount: Amount in quote currency.
+        :param rate: Rate that's going to be used when using limit orders
+        :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
+        :param sell_reason: Sell reason.
+            Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss',
+                           'sell_signal', 'force_sell', 'emergency_sell']
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        :return bool: When True is returned, then the sell-order is placed on the exchange.
+            False aborts the process
+        """
+        if sell_reason == 'force_sell' and trade.calc_profit_ratio(rate) < 0:
+            # Reject force-sells with negative profit
+            # This is just a sample, please adjust to your needs
+            # (this does not necessarily make sense, assuming you know when you're force-selling)
+            return False
+        return True
+
+```
+
+## Derived strategies
+
+The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched:
+
+``` python
+class MyAwesomeStrategy(IStrategy):
+    ...
+    stoploss = 0.13
+    trailing_stop = False
+    # All other attributes and methods are here as they
+    # should be in any custom strategy...
+    ...
+
+class MyAwesomeStrategy2(MyAwesomeStrategy):
+    # Override something
+    stoploss = 0.08
+    trailing_stop = True
+```
+
+Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need.

docs/strategy-customization.md

@@ -1,6 +1,8 @@
 # Strategy Customization
 
-This page explains where to customize your strategies, and add new indicators.
+This page explains how to customize your strategies, add new indicators and set up trading rules.
+
+Please familiarize yourself with [Freqtrade basics](bot-basics.md) first, which provides overall info on how the bot operates.
 
 ## Install a custom strategy file
 
@@ -56,12 +58,12 @@ file as reference.**
 
 !!! Note "Strategies and Backtesting"
     To avoid problems and unexpected differences between Backtesting and dry/live modes, please be aware
-    that during backtesting the full time-interval is passed to the `populate_*()` methods at once.
+    that during backtesting the full time range is passed to the `populate_*()` methods at once.
     It is therefore best to use vectorized operations (across the whole dataframe, not loops) and
     avoid index referencing (`df.iloc[-1]`), but instead use `df.shift()` to get to the previous candle.
 
 !!! Warning "Warning: Using future data"
-    Since backtesting passes the full time interval to the `populate_*()` methods, the strategy author
+    Since backtesting passes the full time range to the `populate_*()` methods, the strategy author
     needs to take care to avoid having the strategy utilize data from the future.
     Some common patterns for this are listed in the [Common Mistakes](#common-mistakes-when-developing-strategies) section of this document.
 
@@ -249,7 +251,7 @@ minimal_roi = {
 While technically not completely disabled, this would sell once the trade reaches 10000% Profit.
 
 To use times based on candle duration (timeframe), the following snippet can be handy.
-This will allow you to change the ticket_interval for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...)
+This will allow you to change the timeframe for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...)
 
 ``` python
 from freqtrade.exchange import timeframe_to_minutes
@@ -283,7 +285,7 @@ If your exchange supports it, it's recommended to also set `"stoploss_on_exchang
 
 For more information on order_types please look [here](configuration.md#understand-order_types).
 
-### Timeframe (ticker interval)
+### Timeframe (formerly ticker interval)
 
 This is the set of candles the bot should download and use for the analysis.
 Common values are `"1m"`, `"5m"`, `"15m"`, `"1h"`, however all values supported by your exchange should work.
@@ -326,15 +328,15 @@ class Awesomestrategy(IStrategy):
 
 ***
 
-### Additional data (informative_pairs)
+## Additional data (informative_pairs)
 
-#### Get data for non-tradeable pairs
+### Get data for non-tradeable pairs
 
 Data for additional, informative pairs (reference pairs) can be beneficial for some strategies.
-Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below).
+OHLCV data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below).
 These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting.
 
-The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument.
+The pairs need to be specified as tuples in the format `("pair", "timeframe")`, with pair as the first and timeframe as the second argument.
 
 Sample:
 
@@ -345,15 +347,17 @@ def informative_pairs(self):
             ]
 ```
 
+A full sample can be found [in the DataProvider section](#complete-data-provider-sample).
+
 !!! Warning
     As these pairs will be refreshed as part of the regular whitelist refresh, it's best to keep this list short.
-    All intervals and all pairs can be specified as long as they are available (and active) on the used exchange.
+    All timeframes and all pairs can be specified as long as they are available (and active) on the used exchange.
-    It is however better to use resampling to longer time-intervals when possible
+    It is however better to use resampling to longer timeframes whenever possible
     to avoid hammering the exchange with too many requests and risk being blocked.
 
 ***
 
-### Additional data (DataProvider)
+## Additional data (DataProvider)
 
 The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy.
 
@@ -361,11 +365,16 @@ All methods return `None` in case of failure (do not raise an exception).
 
 Please always check the mode of operation to select the correct method to get data (samples see below).
 
-#### Possible options for DataProvider
+!!! Warning "Hyperopt"
+    Dataprovider is available during hyperopt, however it can only be used in `populate_indicators()` within a strategy.
+    It is not available in `populate_buy()` and `populate_sell()` methods, nor in `populate_indicators()`, if this method located in the hyperopt file.
 
-- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval).
+### Possible options for DataProvider
-- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist)
+
+- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their timeframe (pair, timeframe).
+- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (i.e. VolumePairlist)
 - [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes).
+- [`get_analyzed_dataframe(pair, timeframe)`](#get_analyzed_dataframepair-timeframe) - Returns the analyzed dataframe (after calling `populate_indicators()`, `populate_buy()`, `populate_sell()`) and the time of the latest analysis.
 - `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk.
 - `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on the Market data structure.
 - `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame.
@@ -373,9 +382,9 @@ Please always check the mode of operation to select the correct method to get da
 - [`ticker(pair)`](#tickerpair) - Returns current ticker data for the pair. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#price-tickers) for more details on the Ticker data structure.
 - `runmode` - Property containing the current runmode.
 
-#### Example Usages:
+### Example Usages
 
-#### *available_pairs*
+### *available_pairs*
 
 ``` python
 if self.dp:
@@ -383,44 +392,31 @@ if self.dp:
         print(f"available {pair}, {timeframe}")
 ```
 
-#### *current_whitelist()*
+### *current_whitelist()*
+
 Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. 
 
 The strategy might look something like this:
 
-*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day ATR to buy and sell.*
+*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day RSI to buy and sell.*
 
-Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least!
+Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day RSI. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least!
 
 Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use.
 
 This is where calling `self.dp.current_whitelist()` comes in handy.
 
 ```python
-class SampleStrategy(IStrategy):
-    # strategy init stuff...
-
-    timeframe = '5m'
-
-    # more strategy init stuff..
-
     def informative_pairs(self):
 
-        # get access to all pairs available in whitelist. 
+        # get access to all pairs available in whitelist.
         pairs = self.dp.current_whitelist()
         # Assign tf to each pair so they can be downloaded and cached for strategy.
         informative_pairs = [(pair, '1d') for pair in pairs]
-        return informative_pairs
+        return informative_pairs   
-
-    def populate_indicators(self, dataframe, metadata):
-        # Get the informative pair
-        informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d')
-        # Get the 14 day ATR.
-        atr = ta.ATR(informative, timeperiod=14)
-        # Do other stuff
 ```
 
-#### *get_pair_dataframe(pair, timeframe)*
+### *get_pair_dataframe(pair, timeframe)*
 
 ``` python
 # fetch live / historical candle (OHLCV) data for the first informative pair
@@ -431,14 +427,27 @@ if self.dp:
 ```
 
 !!! Warning "Warning about backtesting"
-    Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
+    Be careful when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
     for the backtesting runmode) provides the full time-range in one go,
-    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode).
+    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode.
 
-!!! Warning "Warning in hyperopt"
+### *get_analyzed_dataframe(pair, timeframe)*
-    This option cannot currently be used during hyperopt.
 
-#### *orderbook(pair, maximum)*
+This method is used by freqtrade internally to determine the last signal.
+It can also be used in specific callbacks to get the signal that caused the action (see [Advanced Strategy Documentation](strategy-advanced.md) for more details on available callbacks).
+
+``` python
+# fetch current dataframe
+if self.dp:
+    dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'],
+                                                             timeframe=self.timeframe)
+```
+
+!!! Note "No data available"
+    Returns an empty dataframe if the requested pair was not cached.
+    This should not happen when using whitelisted pairs.
+
+### *orderbook(pair, maximum)*
 
 ``` python
 if self.dp:
@@ -449,10 +458,9 @@ if self.dp:
 ```
 
 !!! Warning
-    The order book is not part of the historic data which means backtesting and hyperopt will not work if this
+    The order book is not part of the historic data which means backtesting and hyperopt will not work correctly if this method is used.
-    method is used.
 
-#### *ticker(pair)*
+### *ticker(pair)*
 
 ``` python
 if self.dp:
@@ -469,8 +477,77 @@ if self.dp:
     does not always fills in the `last` field (so it can be None), etc. So you need to carefully verify the ticker
     data returned from the exchange and add appropriate error handling / defaults.
 
+!!! Warning "Warning about backtesting"
+    This method will always return up-to-date values - so usage during backtesting / hyperopt will lead to wrong results.
+
+### Complete Data-provider sample
+
+```python
+class SampleStrategy(IStrategy):
+    # strategy init stuff...
+
+    timeframe = '5m'
+
+    # more strategy init stuff..
+
+    def informative_pairs(self):
+
+        # get access to all pairs available in whitelist.
+        pairs = self.dp.current_whitelist()
+        # Assign tf to each pair so they can be downloaded and cached for strategy.
+        informative_pairs = [(pair, '1d') for pair in pairs]
+        # Optionally Add additional "static" pairs
+        informative_pairs += [("ETH/USDT", "5m"),
+                              ("BTC/TUSD", "15m"),
+                            ]
+        return informative_pairs
+
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        if not self.dp:
+            # Don't do anything if DataProvider is not available.
+            return dataframe
+
+        inf_tf = '1d'
+        # Get the informative pair
+        informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe=inf_tf)
+        # Get the 14 day rsi
+        informative['rsi'] = ta.RSI(informative, timeperiod=14)
+
+        # Rename columns to be unique
+        informative.columns = [f"{col}_{inf_tf}" for col in informative.columns]
+        # Assuming inf_tf = '1d' - then the columns will now be:
+        # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d
+
+        # Combine the 2 dataframes
+        # all indicators on the informative sample MUST be calculated before this point
+        dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_{inf_tf}', how='left')
+        # FFill to have the 1d value available in every row throughout the day.
+        # Without this, comparisons would only work once per day.
+        dataframe = dataframe.ffill()
+
+        # Calculate rsi of the original dataframe (5m timeframe)
+        dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
+
+        # Do other stuff
+        # ...
+
+        return dataframe
+
+    def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+
+        dataframe.loc[
+            (
+                (qtpylib.crossed_above(dataframe['rsi'], 30)) &  # Signal: RSI crosses above 30
+                (dataframe['rsi_1d'] < 30) &                     # Ensure daily RSI is < 30
+                (dataframe['volume'] > 0)                        # Ensure this candle had volume (important for backtesting)
+            ),
+            'buy'] = 1
+
+```
+
 ***
-### Additional data (Wallets)
+
+## Additional data (Wallets)
 
 The strategy provides access to the `Wallets` object. This contains the current balances on the exchange.
 
@@ -486,14 +563,15 @@ if self.wallets:
     total_eth = self.wallets.get_total('ETH')
 ```
 
-#### Possible options for Wallets
+### Possible options for Wallets
 
 - `get_free(asset)` - currently available balance to trade
 - `get_used(asset)` - currently tied up balance (open orders)
 - `get_total(asset)` - total available balance - sum of the 2 above
 
 ***
-### Additional data (Trades)
+
+## Additional data (Trades)
 
 A history of Trades can be retrieved in the strategy by querying the database.
 
@@ -539,13 +617,13 @@ Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5% (ratio of
 !!! Warning
     Trade history is not available during backtesting or hyperopt.
 
-### Prevent trades from happening for a specific pair
+## Prevent trades from happening for a specific pair
 
 Freqtrade locks pairs automatically for the current candle (until that candle is over) when a pair is sold, preventing an immediate re-buy of that pair.
 
 Locked pairs will show the message `Pair <pair> is currently locked.`.
 
-#### Locking pairs from within the strategy
+### Locking pairs from within the strategy
 
 Sometimes it may be desired to lock a pair after certain events happen (e.g. multiple losing trades in a row).
 
@@ -562,7 +640,7 @@ To verify if a pair is currently locked, use `self.is_pair_locked(pair)`.
 !!! Warning
     Locking pairs is not functioning during backtesting.
 
-##### Pair locking example
+#### Pair locking example
 
 ``` python
 from freqtrade.persistence import Trade
@@ -584,7 +662,7 @@ if self.config['runmode'].value in ('live', 'dry_run'):
         self.lock_pair(metadata['pair'], until=datetime.now(timezone.utc) + timedelta(hours=12))
 ```
 
-### Print created dataframe
+## Print created dataframe
 
 To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`.
 You may also want to print the pair so it's clear what data is currently shown.
@@ -608,36 +686,7 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
 
 Printing more than a few rows is also possible (simply use  `print(dataframe)` instead of `print(dataframe.tail())`), however not recommended, as that will be very verbose (~500 lines per pair every 5 seconds).
 
-### Specify custom strategy location
+## Common mistakes when developing strategies
-
-If you want to use a strategy from a different directory you can pass `--strategy-path`
-
-```bash
-freqtrade trade --strategy AwesomeStrategy --strategy-path /some/directory
-```
-
-### Derived strategies
-
-The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched:
-
-``` python
-class MyAwesomeStrategy(IStrategy):
-    ...
-    stoploss = 0.13
-    trailing_stop = False
-    # All other attributes and methods are here as they
-    # should be in any custom strategy...
-    ...
-
-class MyAwesomeStrategy2(MyAwesomeStrategy):
-    # Override something
-    stoploss = 0.08
-    trailing_stop = True
-```
-
-Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need.
-
-### Common mistakes when developing strategies
 
 Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future.
 This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions.
@@ -649,7 +698,7 @@ The following lists some common patterns which should be avoided to prevent frus
 - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling(<window>).mean()` instead
 - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead.
 
-### Further strategy ideas
+## Further strategy ideas
 
 To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk.
 Feel free to use any of them as inspiration for your own strategies.

docs/strategy_analysis_example.md

@@ -85,10 +85,44 @@ Analyze a trades dataframe (also used below for plotting)
 
 
 ```python
-from freqtrade.data.btanalysis import load_backtest_data
+from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats
 
-# Load backtest results
+# if backtest_dir points to a directory, it'll automatically load the last backtest file.
-trades = load_backtest_data(config["user_data_dir"] / "backtest_results/backtest-result.json")
+backtest_dir = config["user_data_dir"] / "backtest_results"
+# backtest_dir can also point to a specific file 
+# backtest_dir = config["user_data_dir"] / "backtest_results/backtest-result-2020-07-01_20-04-22.json"
+```
+
+
+```python
+# You can get the full backtest statistics by using the following command.
+# This contains all information used to generate the backtest result.
+stats = load_backtest_stats(backtest_dir)
+
+strategy = 'SampleStrategy'
+# All statistics are available per strategy, so if `--strategy-list` was used during backtest, this will be reflected here as well.
+# Example usages:
+print(stats['strategy'][strategy]['results_per_pair'])
+# Get pairlist used for this backtest
+print(stats['strategy'][strategy]['pairlist'])
+# Get market change (average change of all pairs from start to end of the backtest period)
+print(stats['strategy'][strategy]['market_change'])
+# Maximum drawdown ()
+print(stats['strategy'][strategy]['max_drawdown'])
+# Maximum drawdown start and end
+print(stats['strategy'][strategy]['drawdown_start'])
+print(stats['strategy'][strategy]['drawdown_end'])
+
+
+# Get strategy comparison (only relevant if multiple strategies were compared)
+print(stats['strategy_comparison'])
+
+```
+
+
+```python
+# Load backtested trades as dataframe
+trades = load_backtest_data(backtest_dir)
 
 # Show value-counts per pair
 trades.groupby("pair")["sell_reason"].value_counts()

docs/telegram-usage.md

@@ -9,7 +9,7 @@ Telegram user id.
 
 Start a chat with the [Telegram BotFather](https://telegram.me/BotFather)
 
-Send the message `/newbot`. 
+Send the message `/newbot`.
 
 *BotFather response:*
 
@@ -47,28 +47,31 @@ Per default, the Telegram bot shows predefined commands. Some commands
 are only available by sending them to the bot. The table below list the
 official commands. You can ask at any moment for help with `/help`.
 
-|  Command | Default | Description |
+|  Command | Description |
-|----------|---------|-------------|
+|----------|-------------|
-| `/start` | | Starts the trader
+| `/start` | Starts the trader
-| `/stop` | | Stops the trader
+| `/stop` | Stops the trader
-| `/stopbuy` | | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
+| `/stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
-| `/reload_config` | | Reloads the configuration file
+| `/reload_config` | Reloads the configuration file
-| `/show_config` | | Shows part of the current configuration with relevant settings to operation
+| `/show_config` | Shows part of the current configuration with relevant settings to operation
-| `/status` | | Lists all open trades
+| `/logs [limit]` | Show last log messages.
-| `/status table` | | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**)
+| `/status` | Lists all open trades
-| `/count` | | Displays number of trades used and available
+| `/status table` | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**)
-| `/profit` | | Display a summary of your profit/loss from close trades and some stats about your performance
+| `/trades [limit]` | List all recently closed trades in a table format.
-| `/forcesell <trade_id>` | | Instantly sells the given trade  (Ignoring `minimum_roi`).
+| `/delete <trade_id>` | Delete a specific trade from the Database. Tries to close open orders. Requires manual handling of this trade on the exchange.
-| `/forcesell all` | | Instantly sells all open trades (Ignoring `minimum_roi`).
+| `/count` | Displays number of trades used and available
-| `/forcebuy <pair> [rate]` | | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True)
+| `/profit` | Display a summary of your profit/loss from close trades and some stats about your performance
-| `/performance` | | Show performance of each finished trade grouped by pair
+| `/forcesell <trade_id>` | Instantly sells the given trade  (Ignoring `minimum_roi`).
-| `/balance` | | Show account balance per currency
+| `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`).
-| `/daily <n>` | 7 | Shows profit or loss per day, over the last n days
+| `/forcebuy <pair> [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True)
-| `/whitelist` | | Show the current whitelist
+| `/performance` | Show performance of each finished trade grouped by pair
-| `/blacklist [pair]` | | Show the current blacklist, or adds a pair to the blacklist.
+| `/balance` | Show account balance per currency
-| `/edge` | | Show validated pairs by Edge if it is enabled.
+| `/daily <n>` | Shows profit or loss per day, over the last n days (n defaults to 7)
-| `/help` | | Show help message
+| `/whitelist` | Show the current whitelist
-| `/version` | | Show version
+| `/blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist.
+| `/edge` | Show validated pairs by Edge if it is enabled.
+| `/help` | Show help message
+| `/version` | Show version
 
 ## Telegram commands in action
 
@@ -113,6 +116,7 @@ For each open trade, the bot will send you the following message.
 ### /status table
 
 Return the status of all open trades in a table format.
+
 ```
    ID  Pair      Since    Profit
 ----  --------  -------  --------
@@ -123,6 +127,7 @@ Return the status of all open trades in a table format.
 ### /count
 
 Return the number of trades used and available.
+
 ```
 current    max
 ---------  -----
@@ -208,7 +213,7 @@ Shows the current whitelist
 
 Shows the current blacklist.
 If Pair is set, then this pair will be added to the pairlist.
-Also supports multiple pairs, seperated by a space.
+Also supports multiple pairs, separated by a space.
 Use `/reload_config` to reset the blacklist.
 
 > Using blacklist `StaticPairList` with 2 pairs  
@@ -216,7 +221,7 @@ Use `/reload_config` to reset the blacklist.
 
 ### /edge
 
-Shows pairs validated by Edge along with their corresponding winrate, expectancy and stoploss values.
+Shows pairs validated by Edge along with their corresponding win-rate, expectancy and stoploss values.
 
 > **Edge only validated following pairs:**
 ```

docs/utils.md

@@ -432,9 +432,9 @@ usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [--max-trades INT] [--min-avg-time FLOAT]
                                [--max-avg-time FLOAT] [--min-avg-profit FLOAT]
                                [--max-avg-profit FLOAT]
-                               [--min-total-profit FLOAT]
+                               [--min-total-profit FLOAT] [--max-total-profit FLOAT]
-                               [--max-total-profit FLOAT] [--no-color]
+                               [--min-objective FLOAT] [--max-objective FLOAT]
-                               [--print-json] [--no-details]
+                               [--no-color] [--print-json] [--no-details]
                                [--export-csv FILE]
 
 optional arguments:
@@ -453,6 +453,10 @@ optional arguments:
                         Select epochs on above total profit.
   --max-total-profit FLOAT
                         Select epochs on below total profit.
+  --min-objective FLOAT
+                        Select epochs on above objective (- is added by default).
+  --max-objective FLOAT
+                        Select epochs on below objective (- is added by default).
   --no-color            Disable colorization of hyperopt results. May be
                         useful if you are redirecting output to a file.
   --print-json          Print best result detailization in JSON format.

docs/webhook-config.md

@@ -47,6 +47,7 @@ Different payloads can be configured for different events. Not all fields are ne
 The fields in `webhook.webhookbuy` are filled when the bot executes a buy. Parameters are filled using string.format.
 Possible parameters are:
 
+* `trade_id`
 * `exchange`
 * `pair`
 * `limit`
@@ -63,6 +64,7 @@ Possible parameters are:
 The fields in `webhook.webhookbuycancel` are filled when the bot cancels a buy order. Parameters are filled using string.format.
 Possible parameters are:
 
+* `trade_id`
 * `exchange`
 * `pair`
 * `limit`
@@ -79,6 +81,7 @@ Possible parameters are:
 The fields in `webhook.webhooksell` are filled when the bot sells a trade. Parameters are filled using string.format.
 Possible parameters are:
 
+* `trade_id`
 * `exchange`
 * `pair`
 * `gain`
@@ -100,6 +103,7 @@ Possible parameters are:
 The fields in `webhook.webhooksellcancel` are filled when the bot cancels a sell order. Parameters are filled using string.format.
 Possible parameters are:
 
+* `trade_id`
 * `exchange`
 * `pair`
 * `gain`

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2020.6'
+__version__ = '2020.8'
 
 if __version__ == 'develop':
 

freqtrade/commands/__init__.py

@@ -9,7 +9,8 @@ Note: Be careful with file-scoped imports in these subfiles.
 from freqtrade.commands.arguments import Arguments
 from freqtrade.commands.build_config_commands import start_new_config
 from freqtrade.commands.data_commands import (start_convert_data,
-                                              start_download_data)
+                                              start_download_data,
+                                              start_list_data)
 from freqtrade.commands.deploy_commands import (start_create_userdir,
                                                 start_new_hyperopt,
                                                 start_new_strategy)

freqtrade/commands/arguments.py

@@ -54,6 +54,8 @@ ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"]
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
+ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
+
 ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
                       "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
 
@@ -71,6 +73,7 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
                       "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time",
                       "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit",
                       "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit",
+                      "hyperopt_list_min_objective", "hyperopt_list_max_objective",
                       "print_colorized", "print_json", "hyperopt_list_no_details",
                       "export_csv"]
 
@@ -78,7 +81,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
                       "print_json", "hyperopt_show_no_header"]
 
 NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
-                    "list-markets", "list-pairs", "list-strategies",
+                    "list-markets", "list-pairs", "list-strategies", "list-data",
                     "list-hyperopts", "hyperopt-list", "hyperopt-show",
                     "plot-dataframe", "plot-profit", "show-trades"]
 
@@ -159,7 +162,7 @@ class Arguments:
         self._build_args(optionlist=['version'], parser=self.parser)
 
         from freqtrade.commands import (start_create_userdir, start_convert_data,
-                                        start_download_data,
+                                        start_download_data, start_list_data,
                                         start_hyperopt_list, start_hyperopt_show,
                                         start_list_exchanges, start_list_hyperopts,
                                         start_list_markets, start_list_strategies,
@@ -233,6 +236,15 @@ class Arguments:
         convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
         self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
 
+        # Add list-data subcommand
+        list_data_cmd = subparsers.add_parser(
+            'list-data',
+            help='List downloaded data.',
+            parents=[_common_parser],
+        )
+        list_data_cmd.set_defaults(func=start_list_data)
+        self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd)
+
         # Add backtesting subcommand
         backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.',
                                                 parents=[_common_parser, _strategy_parser])
@@ -354,7 +366,7 @@ class Arguments:
         plot_profit_cmd = subparsers.add_parser(
             'plot-profit',
             help='Generate plot showing profits.',
-            parents=[_common_parser],
+            parents=[_common_parser, _strategy_parser],
         )
         plot_profit_cmd.set_defaults(func=start_plot_profit)
         self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)

freqtrade/commands/cli_options.py

@@ -455,37 +455,49 @@ AVAILABLE_CLI_OPTIONS = {
     ),
     "hyperopt_list_min_avg_time": Arg(
         '--min-avg-time',
-        help='Select epochs on above average time.',
+        help='Select epochs above average time.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_max_avg_time": Arg(
         '--max-avg-time',
-        help='Select epochs on under average time.',
+        help='Select epochs below average time.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_min_avg_profit": Arg(
         '--min-avg-profit',
-        help='Select epochs on above average profit.',
+        help='Select epochs above average profit.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_max_avg_profit": Arg(
         '--max-avg-profit',
-        help='Select epochs on below average profit.',
+        help='Select epochs below average profit.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_min_total_profit": Arg(
         '--min-total-profit',
-        help='Select epochs on above total profit.',
+        help='Select epochs above total profit.',
         type=float,
         metavar='FLOAT',
     ),
     "hyperopt_list_max_total_profit": Arg(
         '--max-total-profit',
-        help='Select epochs on below total profit.',
+        help='Select epochs below total profit.',
+        type=float,
+        metavar='FLOAT',
+    ),
+    "hyperopt_list_min_objective": Arg(
+        '--min-objective',
+        help='Select epochs above objective.',
+        type=float,
+        metavar='FLOAT',
+    ),
+    "hyperopt_list_max_objective": Arg(
+        '--max-objective',
+        help='Select epochs below objective.',
         type=float,
         metavar='FLOAT',
     ),

freqtrade/commands/data_commands.py

@@ -1,5 +1,6 @@
 import logging
 import sys
+from collections import defaultdict
 from typing import Any, Dict, List
 
 import arrow
@@ -11,6 +12,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv,
                                     refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data)
 from freqtrade.exceptions import OperationalException
+from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.state import RunMode
 
@@ -33,8 +35,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
             "Downloading data requires a list of pairs. "
             "Please check the documentation on how to configure this.")
 
-    logger.info(f'About to download pairs: {config["pairs"]}, '
+    logger.info(f"About to download pairs: {config['pairs']}, "
-                f'intervals: {config["timeframes"]} to {config["datadir"]}')
+                f"intervals: {config['timeframes']} to {config['datadir']}")
 
     pairs_not_available: List[str] = []
 
@@ -49,21 +51,21 @@ def start_download_data(args: Dict[str, Any]) -> None:
 
         if config.get('download_trades'):
             pairs_not_available = refresh_backtest_trades_data(
-                exchange, pairs=config["pairs"], datadir=config['datadir'],
+                exchange, pairs=config['pairs'], datadir=config['datadir'],
-                timerange=timerange, erase=bool(config.get("erase")),
+                timerange=timerange, erase=bool(config.get('erase')),
                 data_format=config['dataformat_trades'])
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
-                pairs=config["pairs"], timeframes=config["timeframes"],
+                pairs=config['pairs'], timeframes=config['timeframes'],
-                datadir=config['datadir'], timerange=timerange, erase=bool(config.get("erase")),
+                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format_ohlcv=config['dataformat_ohlcv'],
                 data_format_trades=config['dataformat_trades'],
                 )
         else:
             pairs_not_available = refresh_backtest_ohlcv_data(
-                exchange, pairs=config["pairs"], timeframes=config["timeframes"],
+                exchange, pairs=config['pairs'], timeframes=config['timeframes'],
-                datadir=config['datadir'], timerange=timerange, erase=bool(config.get("erase")),
+                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format=config['dataformat_ohlcv'])
 
     except KeyboardInterrupt:
@@ -88,3 +90,30 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
         convert_trades_format(config,
                               convert_from=args['format_from'], convert_to=args['format_to'],
                               erase=args['erase'])
+
+
+def start_list_data(args: Dict[str, Any]) -> None:
+    """
+    List available backtest data
+    """
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    from freqtrade.data.history.idatahandler import get_datahandler
+    from tabulate import tabulate
+    dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
+
+    paircombs = dhc.ohlcv_get_available_data(config['datadir'])
+
+    if args['pairs']:
+        paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]
+
+    print(f"Found {len(paircombs)} pair / timeframe combinations.")
+    groupedpair = defaultdict(list)
+    for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))):
+        groupedpair[pair].append(timeframe)
+
+    if groupedpair:
+        print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()],
+                       headers=("Pair", "Timeframe"),
+                       tablefmt='psql', stralign='right'))

freqtrade/commands/deploy_commands.py

@@ -75,7 +75,7 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
         if args["strategy"] == "DefaultStrategy":
             raise OperationalException("DefaultStrategy is not allowed as name.")
 
-        new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args["strategy"] + ".py")
+        new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
 
         if new_path.exists():
             raise OperationalException(f"`{new_path}` already exists. "
@@ -125,11 +125,11 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None:
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    if "hyperopt" in args and args["hyperopt"]:
+    if 'hyperopt' in args and args['hyperopt']:
-        if args["hyperopt"] == "DefaultHyperopt":
+        if args['hyperopt'] == 'DefaultHyperopt':
             raise OperationalException("DefaultHyperopt is not allowed as name.")
 
-        new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py")
+        new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py')
 
         if new_path.exists():
             raise OperationalException(f"`{new_path}` already exists. "

freqtrade/commands/hyperopt_commands.py

@@ -35,7 +35,9 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
         'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
         'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
         'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None)
+        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
+        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
+        'filter_max_objective': config.get('hyperopt_list_max_objective', None),
     }
 
     results_file = (config['user_data_dir'] /
@@ -45,7 +47,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     epochs = Hyperopt.load_previous_results(results_file)
     total_epochs = len(epochs)
 
-    epochs = _hyperopt_filter_epochs(epochs, filteroptions)
+    epochs = hyperopt_filter_epochs(epochs, filteroptions)
 
     if print_colorized:
         colorama_init(autoreset=True)
@@ -92,14 +94,16 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
         'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
         'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
         'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
-        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None)
+        'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
+        'filter_min_objective': config.get('hyperopt_list_min_objective', None),
+        'filter_max_objective': config.get('hyperopt_list_max_objective', None)
     }
 
     # Previous evaluations
     epochs = Hyperopt.load_previous_results(results_file)
     total_epochs = len(epochs)
 
-    epochs = _hyperopt_filter_epochs(epochs, filteroptions)
+    epochs = hyperopt_filter_epochs(epochs, filteroptions)
     filtered_epochs = len(epochs)
 
     if n > filtered_epochs:
@@ -119,7 +123,7 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
                                      header_str="Epoch details")
 
 
-def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
+def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
     """
     Filter our items from the list of hyperopt results
     """
@@ -127,6 +131,24 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
         epochs = [x for x in epochs if x['is_best']]
     if filteroptions['only_profitable']:
         epochs = [x for x in epochs if x['results_metrics']['profit'] > 0]
+
+    epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
+
+    epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
+
+    logger.info(f"{len(epochs)} " +
+                ("best " if filteroptions['only_best'] else "") +
+                ("profitable " if filteroptions['only_profitable'] else "") +
+                "epochs found.")
+    return epochs
+
+
+def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
+
     if filteroptions['filter_min_trades'] > 0:
         epochs = [
             x for x in epochs
@@ -137,6 +159,11 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
             x for x in epochs
             if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades']
         ]
+    return epochs
+
+
+def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
+
     if filteroptions['filter_min_avg_time'] is not None:
         epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
         epochs = [
@@ -149,6 +176,12 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
             x for x in epochs
             if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']
         ]
+
+    return epochs
+
+
+def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
+
     if filteroptions['filter_min_avg_profit'] is not None:
         epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
         epochs = [
@@ -173,10 +206,18 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
             x for x in epochs
             if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit']
         ]
+    return epochs
 
-    logger.info(f"{len(epochs)} " +
+
-                ("best " if filteroptions['only_best'] else "") +
+def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
-                ("profitable " if filteroptions['only_profitable'] else "") +
+
-                "epochs found.")
+    if filteroptions['filter_min_objective'] is not None:
+        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+
+        epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
+    if filteroptions['filter_max_objective'] is not None:
+        epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
+
+        epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
 
     return epochs

freqtrade/commands/list_commands.py

@@ -14,7 +14,7 @@ from freqtrade.configuration import setup_utils_configuration
 from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
-                                market_is_active, symbol_is_pair)
+                                market_is_active)
 from freqtrade.misc import plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.state import RunMode
@@ -163,7 +163,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
             tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'],
                                  'Base': v['base'], 'Quote': v['quote'],
                                  'Active': market_is_active(v),
-                                 **({'Is pair': symbol_is_pair(v['symbol'])}
+                                 **({'Is pair': exchange.market_is_tradable(v)}
                                     if not pairs_only else {})})
 
         if (args.get('print_one_column', False) or

freqtrade/configuration/configuration.py

@@ -54,7 +54,7 @@ class Configuration:
         :param files: List of file paths
         :return: configuration dictionary
         """
-        c = Configuration({"config": files}, RunMode.OTHER)
+        c = Configuration({'config': files}, RunMode.OTHER)
         return c.get_config()
 
     def load_from_files(self, files: List[str]) -> Dict[str, Any]:
@@ -123,10 +123,10 @@ class Configuration:
         the -v/--verbose, --logfile options
         """
         # Log level
-        config.update({'verbosity': self.args.get("verbosity", 0)})
+        config.update({'verbosity': self.args.get('verbosity', 0)})
 
-        if 'logfile' in self.args and self.args["logfile"]:
+        if 'logfile' in self.args and self.args['logfile']:
-            config.update({'logfile': self.args["logfile"]})
+            config.update({'logfile': self.args['logfile']})
 
         setup_logging(config)
 
@@ -149,22 +149,22 @@ class Configuration:
     def _process_common_options(self, config: Dict[str, Any]) -> None:
 
         # Set strategy if not specified in config and or if it's non default
-        if self.args.get("strategy") or not config.get('strategy'):
+        if self.args.get('strategy') or not config.get('strategy'):
-            config.update({'strategy': self.args.get("strategy")})
+            config.update({'strategy': self.args.get('strategy')})
 
         self._args_to_config(config, argname='strategy_path',
                              logstring='Using additional Strategy lookup path: {}')
 
-        if ('db_url' in self.args and self.args["db_url"] and
+        if ('db_url' in self.args and self.args['db_url'] and
-                self.args["db_url"] != constants.DEFAULT_DB_PROD_URL):
+                self.args['db_url'] != constants.DEFAULT_DB_PROD_URL):
-            config.update({'db_url': self.args["db_url"]})
+            config.update({'db_url': self.args['db_url']})
             logger.info('Parameter --db-url detected ...')
 
         if config.get('forcebuy_enable', False):
             logger.warning('`forcebuy` RPC message enabled.')
 
         # Support for sd_notify
-        if 'sd_notify' in self.args and self.args["sd_notify"]:
+        if 'sd_notify' in self.args and self.args['sd_notify']:
             config['internals'].update({'sd_notify': True})
 
     def _process_datadir_options(self, config: Dict[str, Any]) -> None:
@@ -173,24 +173,24 @@ class Configuration:
         --user-data, --datadir
         """
         # Check exchange parameter here - otherwise `datadir` might be wrong.
-        if "exchange" in self.args and self.args["exchange"]:
+        if 'exchange' in self.args and self.args['exchange']:
-            config['exchange']['name'] = self.args["exchange"]
+            config['exchange']['name'] = self.args['exchange']
             logger.info(f"Using exchange {config['exchange']['name']}")
 
         if 'pair_whitelist' not in config['exchange']:
             config['exchange']['pair_whitelist'] = []
 
-        if 'user_data_dir' in self.args and self.args["user_data_dir"]:
+        if 'user_data_dir' in self.args and self.args['user_data_dir']:
-            config.update({'user_data_dir': self.args["user_data_dir"]})
+            config.update({'user_data_dir': self.args['user_data_dir']})
         elif 'user_data_dir' not in config:
             # Default to cwd/user_data (legacy option ...)
-            config.update({'user_data_dir': str(Path.cwd() / "user_data")})
+            config.update({'user_data_dir': str(Path.cwd() / 'user_data')})
 
         # reset to user_data_dir so this contains the absolute path.
         config['user_data_dir'] = create_userdata_dir(config['user_data_dir'], create_dir=False)
         logger.info('Using user-data directory: %s ...', config['user_data_dir'])
 
-        config.update({'datadir': create_datadir(config, self.args.get("datadir", None))})
+        config.update({'datadir': create_datadir(config, self.args.get('datadir', None))})
         logger.info('Using data directory: %s ...', config.get('datadir'))
 
         if self.args.get('exportfilename'):
@@ -199,7 +199,7 @@ class Configuration:
             config['exportfilename'] = Path(config['exportfilename'])
         else:
             config['exportfilename'] = (config['user_data_dir']
-                                        / 'backtest_results/backtest-result.json')
+                                        / 'backtest_results')
 
     def _process_optimize_options(self, config: Dict[str, Any]) -> None:
 
@@ -219,8 +219,8 @@ class Configuration:
             config.update({'use_max_market_positions': False})
             logger.info('Parameter --disable-max-market-positions detected ...')
             logger.info('max_open_trades set to unlimited ...')
-        elif 'max_open_trades' in self.args and self.args["max_open_trades"]:
+        elif 'max_open_trades' in self.args and self.args['max_open_trades']:
-            config.update({'max_open_trades': self.args["max_open_trades"]})
+            config.update({'max_open_trades': self.args['max_open_trades']})
             logger.info('Parameter --max-open-trades detected, '
                         'overriding max_open_trades to: %s ...', config.get('max_open_trades'))
         elif config['runmode'] in NON_UTIL_MODES:
@@ -334,6 +334,12 @@ class Configuration:
         self._args_to_config(config, argname='hyperopt_list_max_total_profit',
                              logstring='Parameter --max-total-profit detected: {}')
 
+        self._args_to_config(config, argname='hyperopt_list_min_objective',
+                             logstring='Parameter --min-objective detected: {}')
+
+        self._args_to_config(config, argname='hyperopt_list_max_objective',
+                             logstring='Parameter --max-objective detected: {}')
+
         self._args_to_config(config, argname='hyperopt_list_no_details',
                              logstring='Parameter --no-details detected: {}')
 
@@ -441,12 +447,12 @@ class Configuration:
                 config['pairs'].sort()
             return
 
-        if "config" in self.args and self.args["config"]:
+        if 'config' in self.args and self.args['config']:
             logger.info("Using pairlist from configuration.")
             config['pairs'] = config.get('exchange', {}).get('pair_whitelist')
         else:
             # Fall back to /dl_path/pairs.json
-            pairs_file = config['datadir'] / "pairs.json"
+            pairs_file = config['datadir'] / 'pairs.json'
             if pairs_file.exists():
                 with pairs_file.open('r') as f:
                     config['pairs'] = json_load(f)

freqtrade/constants.py

@@ -26,12 +26,15 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
                        'ShuffleFilter', 'SpreadFilter']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
 DRY_RUN_WALLET = 1000
+DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons
 DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
 # Don't modify sequence of DEFAULT_TRADES_COLUMNS
 # it has wide consequences for stored trades files
 DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
 
+LAST_BT_RESULT_FN = '.last_result.json'
+
 USERPATH_HYPEROPTS = 'hyperopts'
 USERPATH_STRATEGIES = 'strategies'
 USERPATH_NOTEBOOKS = 'notebooks'
@@ -156,7 +159,9 @@ CONF_SCHEMA = {
                 'emergencysell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
                 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
                 'stoploss_on_exchange': {'type': 'boolean'},
-                'stoploss_on_exchange_interval': {'type': 'number'}
+                'stoploss_on_exchange_interval': {'type': 'number'},
+                'stoploss_on_exchange_limit_ratio': {'type': 'number', 'minimum': 0.0,
+                                                     'maximum': 1.0}
             },
             'required': ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']
         },
@@ -339,4 +344,5 @@ CANCEL_REASON = {
 }
 
 # List of pairs with their timeframes
-ListPairsWithTimeframes = List[Tuple[str, str]]
+PairWithTimeframe = Tuple[str, str]
+ListPairsWithTimeframes = List[PairWithTimeframe]

freqtrade/data/btanalysis.py

@@ -3,52 +3,123 @@ Helpers when analyzing backtest data
 """
 import logging
 from pathlib import Path
-from typing import Dict, Union, Tuple
+from typing import Dict, Union, Tuple, Any, Optional
 
 import numpy as np
 import pandas as pd
 from datetime import timezone
 
 from freqtrade import persistence
+from freqtrade.constants import LAST_BT_RESULT_FN
 from freqtrade.misc import json_load
 from freqtrade.persistence import Trade
 
 logger = logging.getLogger(__name__)
 
 # must align with columns in backtest.py
-BT_DATA_COLUMNS = ["pair", "profit_percent", "open_time", "close_time", "index", "duration",
+BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "trade_duration",
                    "open_rate", "close_rate", "open_at_end", "sell_reason"]
 
 
-def load_backtest_data(filename: Union[Path, str]) -> pd.DataFrame:
+def get_latest_backtest_filename(directory: Union[Path, str]) -> str:
     """
-    Load backtest data file.
+    Get latest backtest export based on '.last_result.json'.
-    :param filename: pathlib.Path object, or string pointing to the file.
+    :param directory: Directory to search for last result
-    :return: a dataframe with the analysis results
+    :return: string containing the filename of the latest backtest result
+    :raises: ValueError in the following cases:
+        * Directory does not exist
+        * `directory/.last_result.json` does not exist
+        * `directory/.last_result.json` has the wrong content
     """
-    if isinstance(filename, str):
+    if isinstance(directory, str):
-        filename = Path(filename)
+        directory = Path(directory)
+    if not directory.is_dir():
+        raise ValueError(f"Directory '{directory}' does not exist.")
+    filename = directory / LAST_BT_RESULT_FN
 
     if not filename.is_file():
-        raise ValueError(f"File {filename} does not exist.")
+        raise ValueError(
+            f"Directory '{directory}' does not seem to contain backtest statistics yet.")
 
     with filename.open() as file:
         data = json_load(file)
 
-    df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)
+    if 'latest_backtest' not in data:
+        raise ValueError(f"Invalid '{LAST_BT_RESULT_FN}' format.")
 
-    df['open_time'] = pd.to_datetime(df['open_time'],
+    return data['latest_backtest']
-                                     unit='s',
+
-                                     utc=True,
+
-                                     infer_datetime_format=True
+def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
-                                     )
+    """
-    df['close_time'] = pd.to_datetime(df['close_time'],
+    Load backtest statistics file.
-                                      unit='s',
+    :param filename: pathlib.Path object, or string pointing to the file.
-                                      utc=True,
+    :return: a dictionary containing the resulting file.
-                                      infer_datetime_format=True
+    """
-                                      )
+    if isinstance(filename, str):
-    df['profit'] = df['close_rate'] - df['open_rate']
+        filename = Path(filename)
-    df = df.sort_values("open_time").reset_index(drop=True)
+    if filename.is_dir():
+        filename = filename / get_latest_backtest_filename(filename)
+    if not filename.is_file():
+        raise ValueError(f"File {filename} does not exist.")
+    logger.info(f"Loading backtest result from {filename}")
+    with filename.open() as file:
+        data = json_load(file)
+
+    return data
+
+
+def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = None) -> pd.DataFrame:
+    """
+    Load backtest data file.
+    :param filename: pathlib.Path object, or string pointing to a file or directory
+    :param strategy: Strategy to load - mainly relevant for multi-strategy backtests
+                     Can also serve as protection to load the correct result.
+    :return: a dataframe with the analysis results
+    :raise: ValueError if loading goes wrong.
+    """
+    data = load_backtest_stats(filename)
+    if not isinstance(data, list):
+        # new, nested format
+        if 'strategy' not in data:
+            raise ValueError("Unknown dataformat.")
+
+        if not strategy:
+            if len(data['strategy']) == 1:
+                strategy = list(data['strategy'].keys())[0]
+            else:
+                raise ValueError("Detected backtest result with more than one strategy. "
+                                 "Please specify a strategy.")
+
+        if strategy not in data['strategy']:
+            raise ValueError(f"Strategy {strategy} not available in the backtest result.")
+
+        data = data['strategy'][strategy]['trades']
+        df = pd.DataFrame(data)
+        df['open_date'] = pd.to_datetime(df['open_date'],
+                                         utc=True,
+                                         infer_datetime_format=True
+                                         )
+        df['close_date'] = pd.to_datetime(df['close_date'],
+                                          utc=True,
+                                          infer_datetime_format=True
+                                          )
+    else:
+        # old format - only with lists.
+        df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)
+
+        df['open_date'] = pd.to_datetime(df['open_date'],
+                                         unit='s',
+                                         utc=True,
+                                         infer_datetime_format=True
+                                         )
+        df['close_date'] = pd.to_datetime(df['close_date'],
+                                          unit='s',
+                                          utc=True,
+                                          infer_datetime_format=True
+                                          )
+        df['profit_abs'] = df['close_rate'] - df['open_rate']
+    df = df.sort_values("open_date").reset_index(drop=True)
     return df
 
 
@@ -62,9 +133,9 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
     """
     from freqtrade.exchange import timeframe_to_minutes
     timeframe_min = timeframe_to_minutes(timeframe)
-    dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time,
+    dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'],
                                      freq=f"{timeframe_min}min"))
-             for row in results[['open_time', 'close_time']].iterrows()]
+             for row in results[['open_date', 'close_date']].iterrows()]
     deltas = [len(x) for x in dates]
     dates = pd.Series(pd.concat(dates).values, name='date')
     df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns)
@@ -90,21 +161,26 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
     return df_final[df_final['open_trades'] > max_open_trades]
 
 
-def load_trades_from_db(db_url: str) -> pd.DataFrame:
+def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataFrame:
     """
     Load trades from a DB (using dburl)
     :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite)
+    :param strategy: Strategy to load - mainly relevant for multi-strategy backtests
+                     Can also serve as protection to load the correct result.
     :return: Dataframe containing Trades
     """
-    trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS)
     persistence.init(db_url, clean_open_orders=False)
 
-    columns = ["pair", "open_time", "close_time", "profit", "profit_percent",
+    columns = ["pair", "open_date", "close_date", "profit", "profit_percent",
-               "open_rate", "close_rate", "amount", "duration", "sell_reason",
+               "open_rate", "close_rate", "amount", "trade_duration", "sell_reason",
                "fee_open", "fee_close", "open_rate_requested", "close_rate_requested",
                "stake_amount", "max_rate", "min_rate", "id", "exchange",
                "stop_loss", "initial_stop_loss", "strategy", "timeframe"]
 
+    filters = []
+    if strategy:
+        filters.append(Trade.strategy == strategy)
+
     trades = pd.DataFrame([(t.pair,
                             t.open_date.replace(tzinfo=timezone.utc),
                             t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None,
@@ -123,16 +199,16 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
                             t.stop_loss, t.initial_stop_loss,
                             t.strategy, t.timeframe
                             )
-                           for t in Trade.get_trades().all()],
+                           for t in Trade.get_trades(filters).all()],
                           columns=columns)
 
     return trades
 
 
 def load_trades(source: str, db_url: str, exportfilename: Path,
-                no_trades: bool = False) -> pd.DataFrame:
+                no_trades: bool = False, strategy: Optional[str] = None) -> pd.DataFrame:
     """
-    Based on configuration option "trade_source":
+    Based on configuration option 'trade_source':
     * loads data from DB (using `db_url`)
     * loads data from backtestfile (using `exportfilename`)
     :param source: "DB" or "file" - specify source to load from
@@ -148,7 +224,7 @@ def load_trades(source: str, db_url: str, exportfilename: Path,
     if source == "DB":
         return load_trades_from_db(db_url)
     elif source == "file":
-        return load_backtest_data(exportfilename)
+        return load_backtest_data(exportfilename, strategy)
 
 
 def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
@@ -163,11 +239,31 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
     else:
         trades_start = dataframe.iloc[0]['date']
         trades_stop = dataframe.iloc[-1]['date']
-    trades = trades.loc[(trades['open_time'] >= trades_start) &
+    trades = trades.loc[(trades['open_date'] >= trades_start) &
-                        (trades['close_time'] <= trades_stop)]
+                        (trades['close_date'] <= trades_stop)]
     return trades
 
 
+def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float:
+    """
+    Calculate market change based on "column".
+    Calculation is done by taking the first non-null and the last non-null element of each column
+    and calculating the pctchange as "(last - first) / first".
+    Then the results per pair are combined as mean.
+
+    :param data: Dict of Dataframes, dict key should be pair.
+    :param column: Column in the original dataframes to use
+    :return:
+    """
+    tmp_means = []
+    for pair, df in data.items():
+        start = df[column].dropna().iloc[0]
+        end = df[column].dropna().iloc[-1]
+        tmp_means.append((end - start) / start)
+
+    return np.mean(tmp_means)
+
+
 def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
                                  column: str = "close") -> pd.DataFrame:
     """
@@ -190,7 +286,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     """
     Adds a column `col_name` with the cumulative profit for the given trades array.
     :param df: DataFrame with date index
-    :param trades: DataFrame containing trades (requires columns close_time and profit_percent)
+    :param trades: DataFrame containing trades (requires columns close_date and profit_percent)
     :param col_name: Column name that will be assigned the results
     :param timeframe: Timeframe used during the operations
     :return: Returns df with one additional column, col_name, containing the cumulative profit.
@@ -201,7 +297,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     from freqtrade.exchange import timeframe_to_minutes
     timeframe_minutes = timeframe_to_minutes(timeframe)
     # Resample to timeframe to make sure trades match candles
-    _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_time'
+    _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
                                   )[['profit_percent']].sum()
     df.loc[:, col_name] = _trades_sum.cumsum()
     # Set first value to 0
@@ -211,13 +307,13 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     return df
 
 
-def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time',
+def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
                            value_col: str = 'profit_percent'
                            ) -> Tuple[float, pd.Timestamp, pd.Timestamp]:
     """
     Calculate max drawdown and the corresponding close dates
-    :param trades: DataFrame containing trades (requires columns close_time and profit_percent)
+    :param trades: DataFrame containing trades (requires columns close_date and profit_percent)
-    :param date_col: Column in DataFrame to use for dates (defaults to 'close_time')
+    :param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
     :param value_col: Column in DataFrame to use for values (defaults to 'profit_percent')
     :return: Tuple (float, highdate, lowdate) with absolute max drawdown, high and low time
     :raise: ValueError if trade-dataframe was found empty.

freqtrade/data/dataprovider.py

@@ -5,16 +5,17 @@ including ticker and orderbook data, live and historical candle (OHLCV) data
 Common Interface for bot and strategy to access data.
 """
 import logging
-from typing import Any, Dict, List, Optional
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, Tuple
 
+from arrow import Arrow
 from pandas import DataFrame
 
+from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe
 from freqtrade.data.history import load_pair_history
-from freqtrade.exceptions import DependencyException, OperationalException
+from freqtrade.exceptions import ExchangeError, OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.state import RunMode
-from freqtrade.constants import ListPairsWithTimeframes
-
 
 logger = logging.getLogger(__name__)
 
@@ -25,6 +26,18 @@ class DataProvider:
         self._config = config
         self._exchange = exchange
         self._pairlists = pairlists
+        self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
+
+    def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None:
+        """
+        Store cached Dataframe.
+        Using private method as this should never be used by a user
+        (but the class is exposed via `self.dp` to the strategy)
+        :param pair: pair to get the data for
+        :param timeframe: Timeframe to get data for
+        :param dataframe: analyzed dataframe
+        """
+        self.__cached_pairs[(pair, timeframe)] = (dataframe, Arrow.utcnow().datetime)
 
     def refresh(self,
                 pairlist: ListPairsWithTimeframes,
@@ -89,6 +102,20 @@ class DataProvider:
             logger.warning(f"No data found for ({pair}, {timeframe}).")
         return data
 
+    def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]:
+        """
+        :param pair: pair to get the data for
+        :param timeframe: timeframe to get data for
+        :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe
+            combination.
+            Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached.
+        """
+        if (pair, timeframe) in self.__cached_pairs:
+            return self.__cached_pairs[(pair, timeframe)]
+        else:
+
+            return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc))
+
     def market(self, pair: str) -> Optional[Dict[str, Any]]:
         """
         Return market data for the pair
@@ -105,7 +132,7 @@ class DataProvider:
         """
         try:
             return self._exchange.fetch_ticker(pair)
-        except DependencyException:
+        except ExchangeError:
             return {}
 
     def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:

freqtrade/data/history/idatahandler.py

@@ -13,6 +13,7 @@ from typing import List, Optional, Type
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
+from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.data.converter import (clean_ohlcv_dataframe,
                                       trades_remove_duplicates, trim_dataframe)
 from freqtrade.exchange import timeframe_to_seconds
@@ -28,6 +29,14 @@ class IDataHandler(ABC):
     def __init__(self, datadir: Path) -> None:
         self._datadir = datadir
 
+    @abstractclassmethod
+    def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Tuples of (pair, timeframe)
+        """
+
     @abstractclassmethod
     def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
         """

freqtrade/data/history/jsondatahandler.py

@@ -8,7 +8,8 @@ from pandas import DataFrame, read_json, to_datetime
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
+from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
+                                 ListPairsWithTimeframes)
 from freqtrade.data.converter import trades_dict_to_list
 
 from .idatahandler import IDataHandler, TradeList
@@ -21,6 +22,18 @@ class JsonDataHandler(IDataHandler):
     _use_zip = False
     _columns = DEFAULT_DATAFRAME_COLUMNS
 
+    @classmethod
+    def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Tuples of (pair, timeframe)
+        """
+        _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name)
+                for p in datadir.glob(f"*.{cls._get_file_extension()}")]
+        return [(match[1].replace('_', '/'), match[2]) for match in _tmp
+                if match and len(match.groups()) > 1]
+
     @classmethod
     def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
         """

freqtrade/edge/edge_positioning.py

@@ -9,7 +9,7 @@ import utils_find_1st as utf1st
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
+from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, DATETIME_PRINT_FORMAT
 from freqtrade.exceptions import OperationalException
 from freqtrade.data.history import get_timerange, load_data, refresh_data
 from freqtrade.strategy.interface import SellType
@@ -121,12 +121,9 @@ class Edge:
 
         # Print timeframe
         min_date, max_date = get_timerange(preprocessed)
-        logger.info(
+        logger.info(f'Measuring data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
-            'Measuring data from %s up to %s (%s days) ...',
+                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-            min_date.isoformat(),
+                    f'({(max_date - min_date).days} days)..')
-            max_date.isoformat(),
-            (max_date - min_date).days
-        )
         headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
 
         trades: list = []
@@ -240,7 +237,7 @@ class Edge:
         # All returned values are relative, they are defined as ratios.
         stake = 0.015
 
-        result['trade_duration'] = result['close_time'] - result['open_time']
+        result['trade_duration'] = result['close_date'] - result['open_date']
 
         result['trade_duration'] = result['trade_duration'].map(
             lambda x: int(x.total_seconds() / 60))
@@ -281,8 +278,8 @@ class Edge:
         #
         # Removing Pumps
         if self.edge_config.get('remove_pumps', False):
-            results = results.groupby(['pair', 'stoploss']).apply(
+            results = results[results['profit_abs'] < 2 * results['profit_abs'].std()
-                lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()])
+                              + results['profit_abs'].mean()]
         ##########################################################################
 
         # Removing trades having a duration more than X minutes (set in config)
@@ -430,10 +427,8 @@ class Edge:
                      'stoploss': stoploss,
                      'profit_ratio': '',
                      'profit_abs': '',
-                     'open_time': date_column[open_trade_index],
+                     'open_date': date_column[open_trade_index],
-                     'close_time': date_column[exit_index],
+                     'close_date': date_column[exit_index],
-                     'open_index': start_point + open_trade_index,
-                     'close_index': start_point + exit_index,
                      'trade_duration': '',
                      'open_rate': round(open_price, 15),
                      'close_rate': round(exit_price, 15),

freqtrade/exceptions.py

@@ -29,7 +29,14 @@ class PricingError(DependencyException):
     """
 
 
-class InvalidOrderException(FreqtradeException):
+class ExchangeError(DependencyException):
+    """
+    Error raised out of the exchange.
+    Has multiple Errors to determine the appropriate error.
+    """
+
+
+class InvalidOrderException(ExchangeError):
     """
     This is returned when the order is not valid. Example:
     If stoploss on exchange order is hit, then trying to cancel the order
@@ -37,7 +44,14 @@ class InvalidOrderException(FreqtradeException):
     """
 
 
-class TemporaryError(FreqtradeException):
+class RetryableOrderError(InvalidOrderException):
+    """
+    This is returned when the order is not found.
+    This Error will be repeated with increasing backof (in line with DDosError).
+    """
+
+
+class TemporaryError(ExchangeError):
     """
     Temporary network or exchange related error.
     This could happen when an exchange is congested, unavailable, or the user
@@ -45,6 +59,13 @@ class TemporaryError(FreqtradeException):
     """
 
 
+class DDosProtection(TemporaryError):
+    """
+    Temporary error caused by DDOS protection.
+    Bot will wait for a second and then retry.
+    """
+
+
 class StrategyError(FreqtradeException):
     """
     Errors with custom user-code deteced.

freqtrade/exchange/__init__.py

@@ -12,8 +12,7 @@ from freqtrade.exchange.exchange import (timeframe_to_seconds,
                                          timeframe_to_msecs,
                                          timeframe_to_next_date,
                                          timeframe_to_prev_date)
-from freqtrade.exchange.exchange import (market_is_active,
+from freqtrade.exchange.exchange import (market_is_active)
-                                         symbol_is_pair)
 from freqtrade.exchange.kraken import Kraken
 from freqtrade.exchange.binance import Binance
 from freqtrade.exchange.bibox import Bibox

freqtrade/exchange/binance.py

@@ -4,9 +4,11 @@ from typing import Dict
 
 import ccxt
 
-from freqtrade.exceptions import (DependencyException, InvalidOrderException,
+from freqtrade.exceptions import (DDosProtection, ExchangeError,
-                                  OperationalException, TemporaryError)
+                                  InvalidOrderException, OperationalException,
+                                  TemporaryError)
 from freqtrade.exchange import Exchange
+from freqtrade.exchange.common import retrier
 
 logger = logging.getLogger(__name__)
 
@@ -39,6 +41,7 @@ class Binance(Exchange):
         """
         return order['type'] == 'stop_loss_limit' and stop_loss > float(order['info']['stopPrice'])
 
+    @retrier(retries=0)
     def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict:
         """
         creates a stoploss limit order.
@@ -77,8 +80,8 @@ class Binance(Exchange):
                         'stop price: %s. limit: %s', pair, stop_price, rate)
             return order
         except ccxt.InsufficientFunds as e:
-            raise DependencyException(
+            raise ExchangeError(
-                f'Insufficient funds to create {ordertype} sell order on market {pair}.'
+                f'Insufficient funds to create {ordertype} sell order on market {pair}. '
                 f'Tried to sell amount {amount} at rate {rate}. '
                 f'Message: {e}') from e
         except ccxt.InvalidOrder as e:
@@ -88,6 +91,8 @@ class Binance(Exchange):
                 f'Could not create {ordertype} sell order on market {pair}. '
                 f'Tried to sell amount {amount} at rate {rate}. '
                 f'Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e

freqtrade/exchange/common.py

@@ -1,6 +1,10 @@
+import asyncio
 import logging
+import time
+from functools import wraps
 
-from freqtrade.exceptions import TemporaryError
+from freqtrade.exceptions import (DDosProtection, RetryableOrderError,
+                                  TemporaryError)
 
 logger = logging.getLogger(__name__)
 
@@ -12,6 +16,7 @@ BAD_EXCHANGES = {
                 "Details in https://github.com/freqtrade/freqtrade/issues/1983",
     "hitbtc": "This API cannot be used with Freqtrade. "
               "Use `hitbtc2` exchange id to access this exchange.",
+    "phemex": "Does not provide history. ",
     **dict.fromkeys([
         'adara',
         'anxpro',
@@ -88,6 +93,13 @@ MAP_EXCHANGE_CHILDCLASS = {
 }
 
 
+def calculate_backoff(retrycount, max_retries):
+    """
+    Calculate backoff
+    """
+    return (max_retries - retrycount) ** 2 + 1
+
+
 def retrier_async(f):
     async def wrapper(*args, **kwargs):
         count = kwargs.pop('count', API_RETRY_COUNT)
@@ -96,9 +108,13 @@ def retrier_async(f):
         except TemporaryError as ex:
             logger.warning('%s() returned exception: "%s"', f.__name__, ex)
             if count > 0:
+                logger.warning('retrying %s() still for %s times', f.__name__, count)
                 count -= 1
                 kwargs.update({'count': count})
-                logger.warning('retrying %s() still for %s times', f.__name__, count)
+                if isinstance(ex, DDosProtection):
+                    backoff_delay = calculate_backoff(count + 1, API_RETRY_COUNT)
+                    logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}")
+                    await asyncio.sleep(backoff_delay)
                 return await wrapper(*args, **kwargs)
             else:
                 logger.warning('Giving up retrying: %s()', f.__name__)
@@ -106,19 +122,31 @@ def retrier_async(f):
     return wrapper
 
 
-def retrier(f):
+def retrier(_func=None, retries=API_RETRY_COUNT):
-    def wrapper(*args, **kwargs):
+    def decorator(f):
-        count = kwargs.pop('count', API_RETRY_COUNT)
+        @wraps(f)
-        try:
+        def wrapper(*args, **kwargs):
-            return f(*args, **kwargs)
+            count = kwargs.pop('count', retries)
-        except TemporaryError as ex:
+            try:
-            logger.warning('%s() returned exception: "%s"', f.__name__, ex)
+                return f(*args, **kwargs)
-            if count > 0:
+            except (TemporaryError, RetryableOrderError) as ex:
-                count -= 1
+                logger.warning('%s() returned exception: "%s"', f.__name__, ex)
-                kwargs.update({'count': count})
+                if count > 0:
-                logger.warning('retrying %s() still for %s times', f.__name__, count)
+                    logger.warning('retrying %s() still for %s times', f.__name__, count)
-                return wrapper(*args, **kwargs)
+                    count -= 1
-            else:
+                    kwargs.update({'count': count})
-                logger.warning('Giving up retrying: %s()', f.__name__)
+                    if isinstance(ex, DDosProtection) or isinstance(ex, RetryableOrderError):
-                raise ex
+                        # increasing backoff
-    return wrapper
+                        backoff_delay = calculate_backoff(count + 1, retries)
+                        logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}")
+                        time.sleep(backoff_delay)
+                    return wrapper(*args, **kwargs)
+                else:
+                    logger.warning('Giving up retrying: %s()', f.__name__)
+                    raise ex
+        return wrapper
+    # Support both @retrier and @retrier(retries=2) syntax
+    if _func is None:
+        return decorator
+    else:
+        return decorator(_func)

freqtrade/exchange/exchange.py

@@ -18,12 +18,13 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE,
                                             TRUNCATE, decimal_to_precision)
 from pandas import DataFrame
 
-from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
-from freqtrade.exceptions import (DependencyException, InvalidOrderException,
-                                  OperationalException, TemporaryError)
-from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
-from freqtrade.misc import deep_merge_dicts, safe_value_fallback
 from freqtrade.constants import ListPairsWithTimeframes
+from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
+from freqtrade.exceptions import (DDosProtection, ExchangeError,
+                                  InvalidOrderException, OperationalException,
+                                  RetryableOrderError, TemporaryError)
+from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
+from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
 
 CcxtModuleType = Any
 
@@ -84,8 +85,8 @@ class Exchange:
 
         # Deep merge ft_has with default ft_has options
         self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default))
-        if exchange_config.get("_ft_has_params"):
+        if exchange_config.get('_ft_has_params'):
-            self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"),
+            self._ft_has = deep_merge_dicts(exchange_config.get('_ft_has_params'),
                                             self._ft_has)
             logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has)
 
@@ -186,6 +187,11 @@ class Exchange:
     def timeframes(self) -> List[str]:
         return list((self._api.timeframes or {}).keys())
 
+    @property
+    def ohlcv_candle_limit(self) -> int:
+        """exchange ohlcv candle limit"""
+        return int(self._ohlcv_candle_limit)
+
     @property
     def markets(self) -> Dict:
         """exchange ccxt markets"""
@@ -216,7 +222,7 @@ class Exchange:
         if quote_currencies:
             markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies}
         if pairs_only:
-            markets = {k: v for k, v in markets.items() if symbol_is_pair(v['symbol'])}
+            markets = {k: v for k, v in markets.items() if self.market_is_tradable(v)}
         if active_only:
             markets = {k: v for k, v in markets.items() if market_is_active(v)}
         return markets
@@ -240,6 +246,19 @@ class Exchange:
         """
         return self.markets.get(pair, {}).get('base', '')
 
+    def market_is_tradable(self, market: Dict[str, Any]) -> bool:
+        """
+        Check if the market symbol is tradable by Freqtrade.
+        By default, checks if it's splittable by `/` and both sides correspond to base / quote
+        """
+        symbol_parts = market['symbol'].split('/')
+        return (len(symbol_parts) == 2 and
+                len(symbol_parts[0]) > 0 and
+                len(symbol_parts[1]) > 0 and
+                symbol_parts[0] == market.get('base') and
+                symbol_parts[1] == market.get('quote')
+                )
+
     def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame:
         if pair_interval in self._klines:
             return self._klines[pair_interval].copy() if copy else self._klines[pair_interval]
@@ -252,8 +271,8 @@ class Exchange:
                 api.urls['api'] = api.urls['test']
                 logger.info("Enabled Sandbox API on %s", name)
             else:
-                logger.warning(name, "No Sandbox URL in CCXT, exiting. "
+                logger.warning(
-                                     "Please check your config.json")
+                    f"No Sandbox URL in CCXT for {name}, exiting. Please check your config.json")
                 raise OperationalException(f'Exchange {name} does not provide a sandbox api')
 
     def _load_async_markets(self, reload: bool = False) -> None:
@@ -351,7 +370,7 @@ class Exchange:
         for pair in [f"{curr_1}/{curr_2}", f"{curr_2}/{curr_1}"]:
             if pair in self.markets and self.markets[pair].get('active'):
                 return pair
-        raise DependencyException(f"Could not combine {curr_1} and {curr_2} to get a valid pair.")
+        raise ExchangeError(f"Could not combine {curr_1} and {curr_2} to get a valid pair.")
 
     def validate_timeframes(self, timeframe: Optional[str]) -> None:
         """
@@ -474,6 +493,7 @@ class Exchange:
             "id": order_id,
             'pair': pair,
             'price': rate,
+            'average': rate,
             'amount': _amount,
             'cost': _amount * rate,
             'type': ordertype,
@@ -518,15 +538,17 @@ class Exchange:
                                           amount, rate_for_order, params)
 
         except ccxt.InsufficientFunds as e:
-            raise DependencyException(
+            raise ExchangeError(
-                f'Insufficient funds to create {ordertype} {side} order on market {pair}.'
+                f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
                 f'Tried to {side} amount {amount} at rate {rate}.'
                 f'Message: {e}') from e
         except ccxt.InvalidOrder as e:
-            raise DependencyException(
+            raise ExchangeError(
-                f'Could not create {ordertype} {side} order on market {pair}.'
+                f'Could not create {ordertype} {side} order on market {pair}. '
-                f'Tried to {side} amount {amount} at rate {rate}.'
+                f'Tried to {side} amount {amount} at rate {rate}. '
                 f'Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not place {side} order due to {e.__class__.__name__}. Message: {e}') from e
@@ -606,6 +628,8 @@ class Exchange:
             balances.pop("used", None)
 
             return balances
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e
@@ -620,6 +644,8 @@ class Exchange:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching tickers in batch. '
                 f'Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not load tickers due to {e.__class__.__name__}. Message: {e}') from e
@@ -630,9 +656,11 @@ class Exchange:
     def fetch_ticker(self, pair: str) -> dict:
         try:
             if pair not in self._api.markets or not self._api.markets[pair].get('active'):
-                raise DependencyException(f"Pair {pair} not available")
+                raise ExchangeError(f"Pair {pair} not available")
             data = self._api.fetch_ticker(pair)
             return data
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e
@@ -766,6 +794,8 @@ class Exchange:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching historical '
                 f'candle (OHLCV) data. Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(f'Could not fetch historical candle (OHLCV) data '
                                  f'for pair {pair} due to {e.__class__.__name__}. '
@@ -802,6 +832,8 @@ class Exchange:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching historical trade data.'
                 f'Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(f'Could not load trade history due to {e.__class__.__name__}. '
                                  f'Message: {e}') from e
@@ -933,7 +965,7 @@ class Exchange:
     def check_order_canceled_empty(self, order: Dict) -> bool:
         """
         Verify if an order has been cancelled without being partially filled
-        :param order: Order dict as returned from get_order()
+        :param order: Order dict as returned from fetch_order()
         :return: True if order has been cancelled without being filled, False otherwise.
         """
         return order.get('status') in ('closed', 'canceled') and order.get('filled') == 0.0
@@ -948,13 +980,15 @@ class Exchange:
         except ccxt.InvalidOrder as e:
             raise InvalidOrderException(
                 f'Could not cancel order. Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    # Assign method to get_stoploss_order to allow easy overriding in other classes
+    # Assign method to cancel_stoploss_order to allow easy overriding in other classes
     cancel_stoploss_order = cancel_order
 
     def is_cancel_order_result_suitable(self, corder) -> bool:
@@ -968,7 +1002,7 @@ class Exchange:
         """
         Cancel order returning a result.
         Creates a fake result if cancel order returns a non-usable result
-        and get_order does not work (certain exchanges don't return cancelled orders)
+        and fetch_order does not work (certain exchanges don't return cancelled orders)
         :param order_id: Orderid to cancel
         :param pair: Pair corresponding to order_id
         :param amount: Amount to use for fake response
@@ -979,17 +1013,17 @@ class Exchange:
             if self.is_cancel_order_result_suitable(corder):
                 return corder
         except InvalidOrderException:
-            logger.warning(f"Could not cancel order {order_id}.")
+            logger.warning(f"Could not cancel order {order_id} for {pair}.")
         try:
-            order = self.get_order(order_id, pair)
+            order = self.fetch_order(order_id, pair)
         except InvalidOrderException:
             logger.warning(f"Could not fetch cancelled order {order_id}.")
             order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
 
         return order
 
-    @retrier
+    @retrier(retries=5)
-    def get_order(self, order_id: str, pair: str) -> Dict:
+    def fetch_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
             try:
                 order = self._dry_run_open_orders[order_id]
@@ -1000,25 +1034,30 @@ class Exchange:
                     f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e
         try:
             return self._api.fetch_order(order_id, pair)
+        except ccxt.OrderNotFound as e:
+            raise RetryableOrderError(
+                f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e
         except ccxt.InvalidOrder as e:
             raise InvalidOrderException(
-                f'Tried to get an invalid order (id: {order_id}). Message: {e}') from e
+                f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    # Assign method to get_stoploss_order to allow easy overriding in other classes
+    # Assign method to fetch_stoploss_order to allow easy overriding in other classes
-    get_stoploss_order = get_order
+    fetch_stoploss_order = fetch_order
 
     @retrier
     def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
         """
-        get order book level 2 from exchange
+        Get L2 order book from exchange.
-
+        Can be limited to a certain amount (if supported).
-        Notes:
+        Returns a dict in the format
-        20180619: bittrex doesnt support limits -.-
+        {'asks': [price, volume], 'bids': [price, volume]}
         """
         try:
 
@@ -1027,6 +1066,8 @@ class Exchange:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching order book.'
                 f'Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e
@@ -1063,7 +1104,8 @@ class Exchange:
             matched_trades = [trade for trade in my_trades if trade['order'] == order_id]
 
             return matched_trades
-
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e
@@ -1080,6 +1122,8 @@ class Exchange:
 
             return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount,
                                            price=price, takerOrMaker=taker_or_maker)['rate']
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e
@@ -1114,7 +1158,7 @@ class Exchange:
         if fee_curr in self.get_pair_base_currency(order['symbol']):
             # Base currency - divide by amount
             return round(
-                order['fee']['cost'] / safe_value_fallback(order, order, 'filled', 'amount'), 8)
+                order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8)
         elif fee_curr in self.get_pair_quote_currency(order['symbol']):
             # Quote currency - divide by cost
             return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None
@@ -1127,9 +1171,9 @@ class Exchange:
                 comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency'])
                 tick = self.fetch_ticker(comb)
 
-                fee_to_quote_rate = safe_value_fallback(tick, tick, 'last', 'ask')
+                fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask')
                 return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8)
-            except DependencyException:
+            except ExchangeError:
                 return None
 
     def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]:
@@ -1142,7 +1186,6 @@ class Exchange:
         return (order['fee']['cost'],
                 order['fee']['currency'],
                 self.calculate_fee_rate(order))
-        # calculate rate ? (order['fee']['cost'] / (order['amount'] * order['price']))
 
 
 def is_exchange_bad(exchange_name: str) -> bool:
@@ -1228,20 +1271,6 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime:
     return datetime.fromtimestamp(new_timestamp, tz=timezone.utc)
 
 
-def symbol_is_pair(market_symbol: str, base_currency: str = None,
-                   quote_currency: str = None) -> bool:
-    """
-    Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the
-    quote currency separated by '/' character. If base_currency and/or quote_currency is passed,
-    it also checks that the symbol contains appropriate base and/or quote currency part before
-    and after the separating character correspondingly.
-    """
-    symbol_parts = market_symbol.split('/')
-    return (len(symbol_parts) == 2 and
-            (symbol_parts[0] == base_currency if base_currency else len(symbol_parts[0]) > 0) and
-            (symbol_parts[1] == quote_currency if quote_currency else len(symbol_parts[1]) > 0))
-
-
 def market_is_active(market: Dict) -> bool:
     """
     Return True if the market is active.

freqtrade/exchange/ftx.py

@@ -1,11 +1,12 @@
 """ FTX exchange subclass """
 import logging
-from typing import Dict
+from typing import Any, Dict
 
 import ccxt
 
-from freqtrade.exceptions import (DependencyException, InvalidOrderException,
+from freqtrade.exceptions import (DDosProtection, ExchangeError,
-                                  OperationalException, TemporaryError)
+                                  InvalidOrderException, OperationalException,
+                                  TemporaryError)
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import retrier
 
@@ -19,6 +20,16 @@ class Ftx(Exchange):
         "ohlcv_candle_limit": 1500,
     }
 
+    def market_is_tradable(self, market: Dict[str, Any]) -> bool:
+        """
+        Check if the market symbol is tradable by Freqtrade.
+        Default checks + check if pair is spot pair (no futures trading yet).
+        """
+        parent_check = super().market_is_tradable(market)
+
+        return (parent_check and
+                market.get('spot', False) is True)
+
     def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
         """
         Verify stop_loss against stoploss-order value (limit or price)
@@ -26,6 +37,7 @@ class Ftx(Exchange):
         """
         return order['type'] == 'stop' and stop_loss > float(order['price'])
 
+    @retrier(retries=0)
     def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict:
         """
         Creates a stoploss order.
@@ -59,7 +71,7 @@ class Ftx(Exchange):
                         'stop price: %s.', pair, stop_price)
             return order
         except ccxt.InsufficientFunds as e:
-            raise DependencyException(
+            raise ExchangeError(
                 f'Insufficient funds to create {ordertype} sell order on market {pair}. '
                 f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
                 f'Message: {e}') from e
@@ -68,14 +80,16 @@ class Ftx(Exchange):
                 f'Could not create {ordertype} sell order on market {pair}. '
                 f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
                 f'Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    @retrier
+    @retrier(retries=5)
-    def get_stoploss_order(self, order_id: str, pair: str) -> Dict:
+    def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
             try:
                 order = self._dry_run_open_orders[order_id]
@@ -96,6 +110,8 @@ class Ftx(Exchange):
         except ccxt.InvalidOrder as e:
             raise InvalidOrderException(
                 f'Tried to get an invalid order (id: {order_id}). Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e
@@ -111,6 +127,8 @@ class Ftx(Exchange):
         except ccxt.InvalidOrder as e:
             raise InvalidOrderException(
                 f'Could not cancel order. Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e

freqtrade/exchange/kraken.py

@@ -1,11 +1,12 @@
 """ Kraken exchange subclass """
 import logging
-from typing import Dict
+from typing import Any, Dict
 
 import ccxt
 
-from freqtrade.exceptions import (DependencyException, InvalidOrderException,
+from freqtrade.exceptions import (DDosProtection, ExchangeError,
-                                  OperationalException, TemporaryError)
+                                  InvalidOrderException, OperationalException,
+                                  TemporaryError)
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import retrier
 
@@ -21,6 +22,16 @@ class Kraken(Exchange):
         "trades_pagination_arg": "since",
     }
 
+    def market_is_tradable(self, market: Dict[str, Any]) -> bool:
+        """
+        Check if the market symbol is tradable by Freqtrade.
+        Default checks + check if pair is darkpool pair.
+        """
+        parent_check = super().market_is_tradable(market)
+
+        return (parent_check and
+                market.get('darkpool', False) is False)
+
     @retrier
     def get_balances(self) -> dict:
         if self._config['dry_run']:
@@ -45,6 +56,8 @@ class Kraken(Exchange):
                 balances[bal]['free'] = balances[bal]['total'] - balances[bal]['used']
 
             return balances
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e
@@ -58,6 +71,7 @@ class Kraken(Exchange):
         """
         return order['type'] == 'stop-loss' and stop_loss > float(order['price'])
 
+    @retrier(retries=0)
     def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict:
         """
         Creates a stoploss market order.
@@ -84,8 +98,8 @@ class Kraken(Exchange):
                         'stop price: %s.', pair, stop_price)
             return order
         except ccxt.InsufficientFunds as e:
-            raise DependencyException(
+            raise ExchangeError(
-                f'Insufficient funds to create {ordertype} sell order on market {pair}.'
+                f'Insufficient funds to create {ordertype} sell order on market {pair}. '
                 f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
                 f'Message: {e}') from e
         except ccxt.InvalidOrder as e:
@@ -93,6 +107,8 @@ class Kraken(Exchange):
                 f'Could not create {ordertype} sell order on market {pair}. '
                 f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
                 f'Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
                 f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e

freqtrade/freqtradebot.py

@@ -11,16 +11,16 @@ from typing import Any, Dict, List, Optional
 
 import arrow
 from cachetools import TTLCache
-from requests.exceptions import RequestException
 
 from freqtrade import __version__, constants, persistence
 from freqtrade.configuration import validate_config_consistency
 from freqtrade.data.converter import order_book_to_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
-from freqtrade.exceptions import DependencyException, InvalidOrderException, PricingError
+from freqtrade.exceptions import (DependencyException, ExchangeError,
+                                  InvalidOrderException, PricingError)
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
-from freqtrade.misc import safe_value_fallback
+from freqtrade.misc import safe_value_fallback, safe_value_fallback2
 from freqtrade.pairlist.pairlistmanager import PairListManager
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -119,6 +119,8 @@ class FreqtradeBot:
         if self.config['cancel_open_orders_on_exit']:
             self.cancel_all_open_orders()
 
+        self.check_for_open_trades()
+
         self.rpc.cleanup()
         persistence.cleanup()
 
@@ -151,6 +153,10 @@ class FreqtradeBot:
         self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist),
                                   self.strategy.informative_pairs())
 
+        strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)()
+
+        self.strategy.analyze(self.active_pair_whitelist)
+
         with self._sell_lock:
             # Check and handle any timed out open orders
             self.check_handle_timedout()
@@ -175,6 +181,24 @@ class FreqtradeBot:
         if self.config['cancel_open_orders_on_exit']:
             self.cancel_all_open_orders()
 
+    def check_for_open_trades(self):
+        """
+        Notify the user when the bot is stopped
+        and there are still open trades active.
+        """
+        open_trades = Trade.get_trades([Trade.is_open == 1]).all()
+
+        if len(open_trades) != 0:
+            msg = {
+                'type': RPCMessageType.WARNING_NOTIFICATION,
+                'status':  f"{len(open_trades)} open trades active.\n\n"
+                           f"Handle these trades manually on {self.exchange.name}, "
+                           f"or '/start' the bot again and use '/stopbuy' "
+                           f"to handle open trades gracefully. \n"
+                           f"{'Trades are simulated.' if self.config['dry_run'] else ''}",
+            }
+            self.rpc.send_msg(msg)
+
     def _refresh_active_whitelist(self, trades: List[Trade] = []) -> List[str]:
         """
         Refresh active whitelist from pairlist or edge and extend it with
@@ -251,7 +275,7 @@ class FreqtradeBot:
             rate = self._buy_rate_cache.get(pair)
             # Check if cache has been invalidated
             if rate:
-                logger.info(f"Using cached buy rate for {pair}.")
+                logger.debug(f"Using cached buy rate for {pair}.")
                 return rate
 
         bid_strategy = self.config.get('bid_strategy', {})
@@ -409,7 +433,9 @@ class FreqtradeBot:
         """
         logger.debug(f"create_trade for pair {pair}")
 
-        if self.strategy.is_pair_locked(pair):
+        analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe)
+        if self.strategy.is_pair_locked(
+                pair, analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None):
             logger.info(f"Pair {pair} is currently locked.")
             return False
 
@@ -420,9 +446,7 @@ class FreqtradeBot:
             return False
 
         # running get_signal on historical data fetched
-        (buy, sell) = self.strategy.get_signal(
+        (buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
-            pair, self.strategy.timeframe,
-            self.dataprovider.ohlcv(pair, self.strategy.timeframe))
 
         if buy and not sell:
             stake_amount = self.get_trade_stake_amount(pair)
@@ -495,6 +519,12 @@ class FreqtradeBot:
 
         amount = stake_amount / buy_limit_requested
         order_type = self.strategy.order_types['buy']
+        if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
+                pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested,
+                time_in_force=time_in_force):
+            logger.info(f"User requested abortion of buying {pair}")
+            return False
+        amount = self.exchange.amount_to_precision(pair, amount)
         order = self.exchange.buy(pair=pair, ordertype=order_type,
                                   amount=amount, rate=buy_limit_requested,
                                   time_in_force=time_in_force)
@@ -503,6 +533,7 @@ class FreqtradeBot:
 
         # we assume the order is executed at the price requested
         buy_limit_filled_price = buy_limit_requested
+        amount_requested = amount
 
         if order_status == 'expired' or order_status == 'rejected':
             order_tif = self.strategy.order_time_in_force['buy']
@@ -523,15 +554,15 @@ class FreqtradeBot:
                                order['filled'], order['amount'], order['remaining']
                                )
                 stake_amount = order['cost']
-                amount = order['amount']
+                amount = safe_value_fallback(order, 'filled', 'amount')
-                buy_limit_filled_price = order['price']
+                buy_limit_filled_price = safe_value_fallback(order, 'average', 'price')
                 order_id = None
 
         # in case of FOK the order may be filled immediately and fully
         elif order_status == 'closed':
             stake_amount = order['cost']
-            amount = order['amount']
+            amount = safe_value_fallback(order, 'filled', 'amount')
-            buy_limit_filled_price = order['price']
+            buy_limit_filled_price = safe_value_fallback(order, 'average', 'price')
 
         # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
         fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker')
@@ -539,6 +570,7 @@ class FreqtradeBot:
             pair=pair,
             stake_amount=stake_amount,
             amount=amount,
+            amount_requested=amount_requested,
             fee_open=fee,
             fee_close=fee,
             open_rate=buy_limit_filled_price,
@@ -569,6 +601,7 @@ class FreqtradeBot:
         Sends rpc notification when a buy occured.
         """
         msg = {
+            'trade_id': trade.id,
             'type': RPCMessageType.BUY_NOTIFICATION,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
@@ -592,6 +625,7 @@ class FreqtradeBot:
         current_rate = self.get_buy_rate(trade.pair, False)
 
         msg = {
+            'trade_id': trade.id,
             'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
@@ -629,7 +663,7 @@ class FreqtradeBot:
                     trades_closed += 1
 
             except DependencyException as exception:
-                logger.warning('Unable to sell trade: %s', exception)
+                logger.warning('Unable to sell trade %s: %s', trade.pair, exception)
 
         # Updating wallets if any trade occured
         if trades_closed:
@@ -660,7 +694,7 @@ class FreqtradeBot:
             rate = self._sell_rate_cache.get(pair)
             # Check if cache has been invalidated
             if rate:
-                logger.info(f"Using cached sell rate for {pair}.")
+                logger.debug(f"Using cached sell rate for {pair}.")
                 return rate
 
         ask_strategy = self.config.get('ask_strategy', {})
@@ -697,9 +731,10 @@ class FreqtradeBot:
 
         if (config_ask_strategy.get('use_sell_signal', True) or
                 config_ask_strategy.get('ignore_roi_if_buy_signal', False)):
-            (buy, sell) = self.strategy.get_signal(
+            analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair,
-                trade.pair, self.strategy.timeframe,
+                                                                      self.strategy.timeframe)
-                self.dataprovider.ohlcv(trade.pair, self.strategy.timeframe))
+
+            (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df)
 
         if config_ask_strategy.get('use_order_book', False):
             order_book_min = config_ask_strategy.get('order_book_min', 1)
@@ -736,7 +771,7 @@ class FreqtradeBot:
         logger.debug('Found no sell signal for %s.', trade)
         return False
 
-    def create_stoploss_order(self, trade: Trade, stop_price: float, rate: float) -> bool:
+    def create_stoploss_order(self, trade: Trade, stop_price: float) -> bool:
         """
         Abstracts creating stoploss orders from the logic.
         Handles errors and updates the trade database object.
@@ -755,7 +790,7 @@ class FreqtradeBot:
             logger.warning('Selling the trade forcefully')
             self.execute_sell(trade, trade.stop_loss, sell_reason=SellType.EMERGENCY_SELL)
 
-        except DependencyException:
+        except ExchangeError:
             trade.stoploss_order_id = None
             logger.exception('Unable to place a stoploss order on exchange.')
         return False
@@ -773,8 +808,8 @@ class FreqtradeBot:
 
         try:
             # First we check if there is already a stoploss on exchange
-            stoploss_order = self.exchange.get_stoploss_order(trade.stoploss_order_id, trade.pair) \
+            stoploss_order = self.exchange.fetch_stoploss_order(
-                if trade.stoploss_order_id else None
+                trade.stoploss_order_id, trade.pair) if trade.stoploss_order_id else None
         except InvalidOrderException as exception:
             logger.warning('Unable to fetch stoploss order: %s', exception)
 
@@ -795,20 +830,17 @@ class FreqtradeBot:
             return False
 
         # If buy order is fulfilled but there is no stoploss, we add a stoploss on exchange
-        if (not stoploss_order):
+        if not stoploss_order:
-
             stoploss = self.edge.stoploss(pair=trade.pair) if self.edge else self.strategy.stoploss
-
             stop_price = trade.open_rate * (1 + stoploss)
 
-            if self.create_stoploss_order(trade=trade, stop_price=stop_price, rate=stop_price):
+            if self.create_stoploss_order(trade=trade, stop_price=stop_price):
                 trade.stoploss_last_update = datetime.now()
                 return False
 
         # If stoploss order is canceled for some reason we add it
         if stoploss_order and stoploss_order['status'] in ('canceled', 'cancelled'):
-            if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss,
+            if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss):
-                                          rate=trade.stop_loss):
                 return False
             else:
                 trade.stoploss_order_id = None
@@ -845,8 +877,7 @@ class FreqtradeBot:
                                      f"for pair {trade.pair}")
 
                 # Create new stoploss order
-                if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss,
+                if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss):
-                                                  rate=trade.stop_loss):
                     logger.warning(f"Could not create trailing stoploss order "
                                    f"for pair {trade.pair}.")
 
@@ -890,8 +921,8 @@ class FreqtradeBot:
             try:
                 if not trade.open_order_id:
                     continue
-                order = self.exchange.get_order(trade.open_order_id, trade.pair)
+                order = self.exchange.fetch_order(trade.open_order_id, trade.pair)
-            except (RequestException, DependencyException, InvalidOrderException):
+            except (ExchangeError):
                 logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
                 continue
 
@@ -923,8 +954,8 @@ class FreqtradeBot:
 
         for trade in Trade.get_open_order_trades():
             try:
-                order = self.exchange.get_order(trade.open_order_id, trade.pair)
+                order = self.exchange.fetch_order(trade.open_order_id, trade.pair)
-            except (DependencyException, InvalidOrderException):
+            except (ExchangeError):
                 logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
                 continue
 
@@ -946,6 +977,12 @@ class FreqtradeBot:
             reason = constants.CANCEL_REASON['TIMEOUT']
             corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
                                                             trade.amount)
+            # Avoid race condition where the order could not be cancelled coz its already filled.
+            # Simply bailing here is the only safe way - as this order will then be
+            # handled in the next iteration.
+            if corder.get('status') not in ('canceled', 'closed'):
+                logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.")
+                return False
         else:
             # Order was cancelled already, so we can reuse the existing dict
             corder = order
@@ -954,7 +991,7 @@ class FreqtradeBot:
         logger.info('Buy order %s for %s.', reason, trade)
 
         # Using filled to determine the filled amount
-        filled_amount = safe_value_fallback(corder, order, 'filled', 'filled')
+        filled_amount = safe_value_fallback2(corder, order, 'filled', 'filled')
 
         if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
             logger.info('Buy order fully cancelled. Removing %s from database.', trade)
@@ -1077,12 +1114,20 @@ class FreqtradeBot:
             order_type = self.strategy.order_types.get("emergencysell", "market")
 
         amount = self._safe_sell_amount(trade.pair, trade.amount)
+        time_in_force = self.strategy.order_time_in_force['sell']
+
+        if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
+                pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit,
+                time_in_force=time_in_force,
+                sell_reason=sell_reason.value):
+            logger.info(f"User requested abortion of selling {trade.pair}")
+            return False
 
         # Execute sell and update trade record
         order = self.exchange.sell(pair=str(trade.pair),
                                    ordertype=order_type,
                                    amount=amount, rate=limit,
-                                   time_in_force=self.strategy.order_time_in_force['sell']
+                                   time_in_force=time_in_force
                                    )
 
         trade.open_order_id = order['id']
@@ -1113,6 +1158,7 @@ class FreqtradeBot:
 
         msg = {
             'type': RPCMessageType.SELL_NOTIFICATION,
+            'trade_id': trade.id,
             'exchange': trade.exchange.capitalize(),
             'pair': trade.pair,
             'gain': gain,
@@ -1155,6 +1201,7 @@ class FreqtradeBot:
 
         msg = {
             'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
+            'trade_id': trade.id,
             'exchange': trade.exchange.capitalize(),
             'pair': trade.pair,
             'gain': gain,
@@ -1202,14 +1249,15 @@ class FreqtradeBot:
         # Update trade with order values
         logger.info('Found open order for %s', trade)
         try:
-            order = action_order or self.exchange.get_order(order_id, trade.pair)
+            order = action_order or self.exchange.fetch_order(order_id, trade.pair)
         except InvalidOrderException as exception:
             logger.warning('Unable to fetch order %s: %s', order_id, exception)
             return False
         # Try update amount (binance-fix)
         try:
             new_amount = self.get_real_amount(trade, order, order_amount)
-            if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
+            if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount,
+                           abs_tol=constants.MATH_CLOSE_PREC):
                 order['amount'] = new_amount
                 order.pop('filled', None)
                 trade.recalc_open_trade_price()
@@ -1255,7 +1303,7 @@ class FreqtradeBot:
         """
         # Init variables
         if order_amount is None:
-            order_amount = order['amount']
+            order_amount = safe_value_fallback(order, 'filled', 'amount')
         # Only run for closed orders
         if trade.fee_updated(order.get('side', '')) or order['status'] == 'open':
             return order_amount

freqtrade/loggers.py

@@ -1,14 +1,18 @@
 import logging
 import sys
-
 from logging import Formatter
-from logging.handlers import RotatingFileHandler, SysLogHandler
+from logging.handlers import (BufferingHandler, RotatingFileHandler,
-from typing import Any, Dict, List
+                              SysLogHandler)
+from typing import Any, Dict
 
 from freqtrade.exceptions import OperationalException
 
-
 logger = logging.getLogger(__name__)
+LOGFORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+
+# Initialize bufferhandler - will be used for /log endpoints
+bufferHandler = BufferingHandler(1000)
+bufferHandler.setFormatter(Formatter(LOGFORMAT))
 
 
 def _set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None:
@@ -33,17 +37,31 @@ def _set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None:
     )
 
 
+def setup_logging_pre() -> None:
+    """
+    Early setup for logging.
+    Uses INFO loglevel and only the Streamhandler.
+    Early messages (before proper logging setup) will therefore only be sent to additional
+    logging handlers after the real initialization, because we don't know which
+    ones the user desires beforehand.
+    """
+    logging.basicConfig(
+        level=logging.INFO,
+        format=LOGFORMAT,
+        handlers=[logging.StreamHandler(sys.stderr), bufferHandler]
+    )
+
+
 def setup_logging(config: Dict[str, Any]) -> None:
     """
     Process -v/--verbose, --logfile options
     """
     # Log level
     verbosity = config['verbosity']
-
+    logging.root.addHandler(bufferHandler)
-    # Log to stderr
-    log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)]
 
     logfile = config.get('logfile')
+
     if logfile:
         s = logfile.split(':')
         if s[0] == 'syslog':
@@ -58,28 +76,27 @@ def setup_logging(config: Dict[str, Any]) -> None:
             # to perform reduction of repeating messages if this is set in the
             # syslog config. The messages should be equal for this.
             handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
-            log_handlers.append(handler)
+            logging.root.addHandler(handler)
         elif s[0] == 'journald':
             try:
                 from systemd.journal import JournaldLogHandler
             except ImportError:
                 raise OperationalException("You need the systemd python package be installed in "
                                            "order to use logging to journald.")
-            handler = JournaldLogHandler()
+            handler_jd = JournaldLogHandler()
             # No datetime field for logging into journald, to allow syslog
             # to perform reduction of repeating messages if this is set in the
             # syslog config. The messages should be equal for this.
-            handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
+            handler_jd.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
-            log_handlers.append(handler)
+            logging.root.addHandler(handler_jd)
         else:
-            log_handlers.append(RotatingFileHandler(logfile,
+            handler_rf = RotatingFileHandler(logfile,
-                                                    maxBytes=1024 * 1024,  # 1Mb
+                                             maxBytes=1024 * 1024 * 10,  # 10Mb
-                                                    backupCount=10))
+                                             backupCount=10)
+            handler_rf.setFormatter(Formatter(LOGFORMAT))
+            logging.root.addHandler(handler_rf)
 
-    logging.basicConfig(
+    logging.root.setLevel(logging.INFO if verbosity < 1 else logging.DEBUG)
-        level=logging.INFO if verbosity < 1 else logging.DEBUG,
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        handlers=log_handlers
-    )
     _set_loggers(verbosity, config.get('api_server', {}).get('verbosity', 'info'))
+
     logger.info('Verbosity set to %s', verbosity)

freqtrade/main.py

@@ -3,18 +3,17 @@
 Main Freqtrade bot script.
 Read the documentation to know what cli arguments you need.
 """
-
+import logging
-from freqtrade.exceptions import FreqtradeException, OperationalException
 import sys
+from typing import Any, List
+
 # check min. python version
 if sys.version_info < (3, 6):
     sys.exit("Freqtrade requires Python version >= 3.6")
 
-# flake8: noqa E402
-import logging
-from typing import Any, List
-
 from freqtrade.commands import Arguments
+from freqtrade.exceptions import FreqtradeException, OperationalException
+from freqtrade.loggers import setup_logging_pre
 
 
 logger = logging.getLogger('freqtrade')
@@ -28,6 +27,7 @@ def main(sysargv: List[str] = None) -> None:
 
     return_code: Any = 1
     try:
+        setup_logging_pre()
         arguments = Arguments(sysargv)
         args = arguments.get_parsed_arg()
 

freqtrade/misc.py

@@ -134,7 +134,21 @@ def round_dict(d, n):
     return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()}
 
 
-def safe_value_fallback(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None):
+def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None):
+    """
+    Search a value in obj, return this if it's not None.
+    Then search key2 in obj - return that if it's not none - then use default_value.
+    Else falls back to None.
+    """
+    if key1 in obj and obj[key1] is not None:
+        return obj[key1]
+    else:
+        if key2 in obj and obj[key2] is not None:
+            return obj[key2]
+    return default_value
+
+
+def safe_value_fallback2(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None):
     """
     Search a value in dict1, return this if it's not None.
     Fall back to dict2 - return key2 from dict2 if it's not None.

freqtrade/optimize/backtesting.py

@@ -13,6 +13,7 @@ from pandas import DataFrame
 
 from freqtrade.configuration import (TimeRange, remove_credentials,
                                      validate_config_consistency)
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
 from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.dataprovider import DataProvider
@@ -20,11 +21,10 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats,
                                                  show_backtest_results,
-                                                 store_backtest_result)
+                                                 store_backtest_stats)
 from freqtrade.pairlist.pairlistmanager import PairListManager
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.state import RunMode
 from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
 
 logger = logging.getLogger(__name__)
@@ -37,14 +37,15 @@ class BacktestResult(NamedTuple):
     pair: str
     profit_percent: float
     profit_abs: float
-    open_time: datetime
+    open_date: datetime
-    close_time: datetime
+    open_rate: float
-    open_index: int
+    open_fee: float
-    close_index: int
+    close_date: datetime
+    close_rate: float
+    close_fee: float
+    amount: float
     trade_duration: float
     open_at_end: bool
-    open_rate: float
-    close_rate: float
     sell_reason: SellType
 
 
@@ -65,9 +66,8 @@ class Backtesting:
         self.strategylist: List[IStrategy] = []
         self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
 
-        if self.config.get('runmode') != RunMode.HYPEROPT:
+        dataprovider = DataProvider(self.config, self.exchange)
-            self.dataprovider = DataProvider(self.config, self.exchange)
+        IStrategy.dp = dataprovider
-            IStrategy.dp = self.dataprovider
 
         if self.config.get('strategy_list', None):
             for strat in list(self.config['strategy_list']):
@@ -101,7 +101,7 @@ class Backtesting:
         if len(self.pairlists.whitelist) == 0:
             raise OperationalException("No pair in whitelist.")
 
-        if config.get('fee'):
+        if config.get('fee', None) is not None:
             self.fee = config['fee']
         else:
             self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0])
@@ -137,10 +137,10 @@ class Backtesting:
 
         min_date, max_date = history.get_timerange(data)
 
-        logger.info(
+        logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
-            'Loading data from %s up to %s (%s days)..',
+                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
+                    f'({(max_date - min_date).days} days)..')
-        )
+
         # Adjust startts forward if not enough data is available
         timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
                                             self.required_startup, min_date)
@@ -225,7 +225,7 @@ class Backtesting:
             open_rate=buy_row.open,
             open_date=buy_row.date,
             stake_amount=stake_amount,
-            amount=stake_amount / buy_row.open,
+            amount=round(stake_amount / buy_row.open, 8),
             fee_open=self.fee,
             fee_close=self.fee,
             is_open=True,
@@ -246,14 +246,15 @@ class Backtesting:
                 return BacktestResult(pair=pair,
                                       profit_percent=trade.calc_profit_ratio(rate=closerate),
                                       profit_abs=trade.calc_profit(rate=closerate),
-                                      open_time=buy_row.date,
+                                      open_date=buy_row.date,
-                                      close_time=sell_row.date,
-                                      trade_duration=trade_dur,
-                                      open_index=buy_row.Index,
-                                      close_index=sell_row.Index,
-                                      open_at_end=False,
                                       open_rate=buy_row.open,
+                                      open_fee=self.fee,
+                                      close_date=sell_row.date,
                                       close_rate=closerate,
+                                      close_fee=self.fee,
+                                      amount=trade.amount,
+                                      trade_duration=trade_dur,
+                                      open_at_end=False,
                                       sell_reason=sell.sell_type
                                       )
         if partial_ohlcv:
@@ -262,15 +263,16 @@ class Backtesting:
             bt_res = BacktestResult(pair=pair,
                                     profit_percent=trade.calc_profit_ratio(rate=sell_row.open),
                                     profit_abs=trade.calc_profit(rate=sell_row.open),
-                                    open_time=buy_row.date,
+                                    open_date=buy_row.date,
-                                    close_time=sell_row.date,
+                                    open_rate=buy_row.open,
+                                    open_fee=self.fee,
+                                    close_date=sell_row.date,
+                                    close_rate=sell_row.open,
+                                    close_fee=self.fee,
+                                    amount=trade.amount,
                                     trade_duration=int((
                                         sell_row.date - buy_row.date).total_seconds() // 60),
-                                    open_index=buy_row.Index,
-                                    close_index=sell_row.Index,
                                     open_at_end=True,
-                                    open_rate=buy_row.open,
-                                    close_rate=sell_row.open,
                                     sell_reason=SellType.FORCE_SELL
                                     )
             logger.debug(f"{pair} - Force selling still open trade, "
@@ -356,8 +358,8 @@ class Backtesting:
 
                 if trade_entry:
                     logger.debug(f"{pair} - Locking pair till "
-                                 f"close_time={trade_entry.close_time}")
+                                 f"close_date={trade_entry.close_date}")
-                    lock_pair_until[pair] = trade_entry.close_time
+                    lock_pair_until[pair] = trade_entry.close_date
                     trades.append(trade_entry)
                 else:
                     # Set lock_pair_until to end of testing period if trade could not be closed
@@ -400,10 +402,9 @@ class Backtesting:
                 preprocessed[pair] = trim_dataframe(df, timerange)
             min_date, max_date = history.get_timerange(preprocessed)
 
-            logger.info(
+            logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
-                'Backtesting with data from %s up to %s (%s days)..',
+                        f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
+                        f'({(max_date - min_date).days} days)..')
-            )
             # Execute backtest and print results
             all_results[self.strategy.get_strategy_name()] = self.backtest(
                 processed=preprocessed,
@@ -414,8 +415,10 @@ class Backtesting:
                 position_stacking=position_stacking,
             )
 
+        stats = generate_backtest_stats(self.config, data, all_results,
+                                        min_date=min_date, max_date=max_date)
         if self.config.get('export', False):
-            store_backtest_result(self.config['exportfilename'], all_results)
+            store_backtest_stats(self.config['exportfilename'], stats)
+
         # Show backtest results
-        stats = generate_backtest_stats(self.config, data, all_results)
         show_backtest_results(self.config, stats)

freqtrade/optimize/hyperopt.py

@@ -4,27 +4,28 @@
 This module contains the hyperopt logic
 """
 
+import io
 import locale
 import logging
 import random
 import warnings
-from math import ceil
 from collections import OrderedDict
+from math import ceil
 from operator import itemgetter
 from pathlib import Path
 from pprint import pformat
 from typing import Any, Dict, List, Optional
 
+import progressbar
 import rapidjson
+import tabulate
 from colorama import Fore, Style
+from colorama import init as colorama_init
 from joblib import (Parallel, cpu_count, delayed, dump, load,
                     wrap_non_picklable_objects)
-from pandas import DataFrame, json_normalize, isna
+from pandas import DataFrame, isna, json_normalize
-import progressbar
-import tabulate
-from os import path
-import io
 
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.history import get_timerange
 from freqtrade.exceptions import OperationalException
@@ -32,9 +33,11 @@ from freqtrade.misc import plural, round_dict
 from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
 from freqtrade.optimize.hyperopt_interface import IHyperOpt  # noqa: F401
-from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss  # noqa: F401
+from freqtrade.optimize.hyperopt_loss_interface import \
+    IHyperOptLoss  # noqa: F401
 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver,
                                                    HyperOptResolver)
+from freqtrade.strategy import IStrategy
 
 # Suppress scikit-learn FutureWarnings from skopt
 with warnings.catch_warnings():
@@ -312,11 +315,16 @@ class Hyperopt:
 
         trials = json_normalize(results, max_level=1)
         trials['Best'] = ''
+        if 'results_metrics.winsdrawslosses' not in trials.columns:
+            # Ensure compatibility with older versions of hyperopt results
+            trials['results_metrics.winsdrawslosses'] = 'N/A'
+
         trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
+                         'results_metrics.winsdrawslosses',
                          'results_metrics.avg_profit', 'results_metrics.total_profit',
                          'results_metrics.profit', 'results_metrics.duration',
                          'loss', 'is_initial_point', 'is_best']]
-        trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit',
+        trials.columns = ['Best', 'Epoch', 'Trades', 'W/D/L', 'Avg profit', 'Total profit',
                           'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
         trials['is_profit'] = False
         trials.loc[trials['is_initial_point'], 'Best'] = '*     '
@@ -390,7 +398,7 @@ class Hyperopt:
             return
 
         # Verification for overwrite
-        if path.isfile(csv_file):
+        if Path(csv_file).is_file():
             logger.error(f"CSV file already exists: {csv_file}")
             return
 
@@ -558,9 +566,17 @@ class Hyperopt:
         }
 
     def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict:
+        wins = len(backtesting_results[backtesting_results.profit_percent > 0])
+        draws = len(backtesting_results[backtesting_results.profit_percent == 0])
+        losses = len(backtesting_results[backtesting_results.profit_percent < 0])
         return {
             'trade_count': len(backtesting_results.index),
+            'wins': wins,
+            'draws': draws,
+            'losses': losses,
+            'winsdrawslosses': f"{wins}/{draws}/{losses}",
             'avg_profit': backtesting_results.profit_percent.mean() * 100.0,
+            'median_profit': backtesting_results.profit_percent.median() * 100.0,
             'total_profit': backtesting_results.profit_abs.sum(),
             'profit': backtesting_results.profit_percent.sum() * 100.0,
             'duration': backtesting_results.trade_duration.mean(),
@@ -572,7 +588,10 @@ class Hyperopt:
         """
         stake_cur = self.config['stake_currency']
         return (f"{results_metrics['trade_count']:6d} trades. "
+                f"{results_metrics['wins']}/{results_metrics['draws']}"
+                f"/{results_metrics['losses']} Wins/Draws/Losses. "
                 f"Avg profit {results_metrics['avg_profit']: 6.2f}%. "
+                f"Median profit {results_metrics['median_profit']: 6.2f}%. "
                 f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} "
                 f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
                 f"Avg duration {results_metrics['duration']:5.1f} min."
@@ -625,15 +644,17 @@ class Hyperopt:
             preprocessed[pair] = trim_dataframe(df, timerange)
         min_date, max_date = get_timerange(data)
 
-        logger.info(
+        logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
-            'Hyperopting with data from %s up to %s (%s days)..',
+                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-            min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
+                    f'({(max_date - min_date).days} days)..')
-        )
+
         dump(preprocessed, self.data_pickle_file)
 
         # We don't need exchange instance anymore while running hyperopt
         self.backtesting.exchange = None  # type: ignore
         self.backtesting.pairlists = None  # type: ignore
+        self.backtesting.strategy.dp = None  # type: ignore
+        IStrategy.dp = None  # type: ignore
 
         self.epochs = self.load_previous_results(self.results_file)
 
@@ -644,6 +665,10 @@ class Hyperopt:
 
         self.dimensions: List[Dimension] = self.hyperopt_space()
         self.opt = self.get_optimizer(self.dimensions, config_jobs)
+
+        if self.print_colorized:
+            colorama_init(autoreset=True)
+
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
                 jobs = parallel._effective_n_jobs()

freqtrade/optimize/hyperopt_loss_sharpe_daily.py

@@ -43,7 +43,7 @@ class SharpeHyperOptLossDaily(IHyperOptLoss):
                              normalize=True)
 
         sum_daily = (
-            results.resample(resample_freq, on='close_time').agg(
+            results.resample(resample_freq, on='close_date').agg(
                 {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0)
         )
 

freqtrade/optimize/hyperopt_loss_sortino_daily.py

@@ -45,7 +45,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss):
                              normalize=True)
 
         sum_daily = (
-            results.resample(resample_freq, on='close_time').agg(
+            results.resample(resample_freq, on='close_date').agg(
                 {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0)
         )
 

freqtrade/optimize/optimize_reports.py

@@ -1,46 +1,40 @@
 import logging
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Dict, List
 
+from arrow import Arrow
 from pandas import DataFrame
+from numpy import int64
 from tabulate import tabulate
 
+from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
+from freqtrade.data.btanalysis import calculate_max_drawdown, calculate_market_change
 from freqtrade.misc import file_dump_json
 
 logger = logging.getLogger(__name__)
 
 
-def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame]) -> None:
+def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> None:
     """
-    Stores backtest results to file (one file per strategy)
+    Stores backtest results
-    :param recordfilename: Destination filename
+    :param recordfilename: Path object, which can either be a filename or a directory.
-    :param all_results: Dict of Dataframes, one results dataframe per strategy
+        Filenames will be appended with a timestamp right before the suffix
+        while for diectories, <directory>/backtest-result-<datetime>.json will be used as filename
+    :param stats: Dataframe containing the backtesting statistics
     """
-    for strategy, results in all_results.items():
+    if recordfilename.is_dir():
-        records = backtest_result_to_list(results)
+        filename = (recordfilename /
+                    f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.json')
+    else:
+        filename = Path.joinpath(
+            recordfilename.parent,
+            f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
+            ).with_suffix(recordfilename.suffix)
+    file_dump_json(filename, stats)
 
-        if records:
+    latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN)
-            filename = recordfilename
+    file_dump_json(latest_filename, {'latest_backtest': str(filename.name)})
-            if len(all_results) > 1:
-                # Inject strategy to filename
-                filename = Path.joinpath(
-                    recordfilename.parent,
-                    f'{recordfilename.stem}-{strategy}').with_suffix(recordfilename.suffix)
-            logger.info(f'Dumping backtest results to {filename}')
-            file_dump_json(filename, records)
-
-
-def backtest_result_to_list(results: DataFrame) -> List[List]:
-    """
-    Converts a list of Backtest-results to list
-    :param results: Dataframe containing results for one strategy
-    :return: List of Lists containing the trades
-    """
-    return [[t.pair, t.profit_percent, t.open_time.timestamp(),
-             t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
-             t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value]
-            for index, t in results.iterrows()]
 
 
 def _get_line_floatfmt() -> List[str]:
@@ -66,11 +60,12 @@ def _generate_result_line(result: DataFrame, max_open_trades: int, first_column:
     return {
         'key': first_column,
         'trades': len(result),
-        'profit_mean': result['profit_percent'].mean(),
+        'profit_mean': result['profit_percent'].mean() if len(result) > 0 else 0.0,
-        'profit_mean_pct': result['profit_percent'].mean() * 100.0,
+        'profit_mean_pct': result['profit_percent'].mean() * 100.0 if len(result) > 0 else 0.0,
         'profit_sum': result['profit_percent'].sum(),
         'profit_sum_pct': result['profit_percent'].sum() * 100.0,
         'profit_total_abs': result['profit_abs'].sum(),
+        'profit_total': result['profit_percent'].sum() / max_open_trades,
         'profit_total_pct': result['profit_percent'].sum() * 100.0 / max_open_trades,
         'duration_avg': str(timedelta(
                             minutes=round(result['trade_duration'].mean()))
@@ -141,7 +136,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
                 'profit_sum': profit_sum,
                 'profit_sum_pct': round(profit_sum * 100, 2),
                 'profit_total_abs': result['profit_abs'].sum(),
-                'profit_pct_total': profit_percent_tot,
+                'profit_total_pct': profit_percent_tot,
             }
         )
     return tabular_data
@@ -189,18 +184,58 @@ def generate_edge_table(results: dict) -> str:
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")  # type: ignore
 
 
+def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
+    if len(results) == 0:
+        return {
+            'backtest_best_day': 0,
+            'backtest_worst_day': 0,
+            'winning_days': 0,
+            'draw_days': 0,
+            'losing_days': 0,
+            'winner_holding_avg': timedelta(),
+            'loser_holding_avg': timedelta(),
+        }
+    daily_profit = results.resample('1d', on='close_date')['profit_percent'].sum()
+    worst = min(daily_profit)
+    best = max(daily_profit)
+    winning_days = sum(daily_profit > 0)
+    draw_days = sum(daily_profit == 0)
+    losing_days = sum(daily_profit < 0)
+
+    winning_trades = results.loc[results['profit_percent'] > 0]
+    losing_trades = results.loc[results['profit_percent'] < 0]
+
+    return {
+        'backtest_best_day': best,
+        'backtest_worst_day': worst,
+        'winning_days': winning_days,
+        'draw_days': draw_days,
+        'losing_days': losing_days,
+        'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
+                               if not winning_trades.empty else timedelta()),
+        'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
+                              if not losing_trades.empty else timedelta()),
+    }
+
+
 def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
-                            all_results: Dict[str, DataFrame]) -> Dict[str, Any]:
+                            all_results: Dict[str, DataFrame],
+                            min_date: Arrow, max_date: Arrow
+                            ) -> Dict[str, Any]:
     """
     :param config: Configuration object used for backtest
     :param btdata: Backtest data
     :param all_results: backtest result - dictionary with { Strategy: results}.
+    :param min_date: Backtest start date
+    :param max_date: Backtest end date
     :return:
     Dictionary containing results per strategy and a stratgy summary.
     """
     stake_currency = config['stake_currency']
     max_open_trades = config['max_open_trades']
     result: Dict[str, Any] = {'strategy': {}}
+    market_change = calculate_market_change(btdata, 'close')
+
     for strategy, results in all_results.items():
 
         pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
@@ -212,14 +247,58 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
                                                   max_open_trades=max_open_trades,
                                                   results=results.loc[results['open_at_end']],
                                                   skip_nan=True)
+        daily_stats = generate_daily_stats(results)
+
+        results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
+        results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
+
+        backtest_days = (max_date - min_date).days
         strat_stats = {
-            'trades': backtest_result_to_list(results),
+            'trades': results.to_dict(orient='records'),
             'results_per_pair': pair_results,
             'sell_reason_summary': sell_reason_stats,
             'left_open_trades': left_open_results,
-            }
+            'total_trades': len(results),
+            'profit_mean': results['profit_percent'].mean() if len(results) > 0 else 0,
+            'profit_total': results['profit_percent'].sum(),
+            'profit_total_abs': results['profit_abs'].sum(),
+            'backtest_start': min_date.datetime,
+            'backtest_start_ts': min_date.timestamp * 1000,
+            'backtest_end': max_date.datetime,
+            'backtest_end_ts': max_date.timestamp * 1000,
+            'backtest_days': backtest_days,
+
+            'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
+            'market_change': market_change,
+            'pairlist': list(btdata.keys()),
+            'stake_amount': config['stake_amount'],
+            'stake_currency': config['stake_currency'],
+            'max_open_trades': (config['max_open_trades']
+                                if config['max_open_trades'] != float('inf') else -1),
+            'timeframe': config['timeframe'],
+            **daily_stats,
+        }
         result['strategy'][strategy] = strat_stats
 
+        try:
+            max_drawdown, drawdown_start, drawdown_end = calculate_max_drawdown(
+                results, value_col='profit_percent')
+            strat_stats.update({
+                'max_drawdown': max_drawdown,
+                'drawdown_start': drawdown_start,
+                'drawdown_start_ts': drawdown_start.timestamp() * 1000,
+                'drawdown_end': drawdown_end,
+                'drawdown_end_ts': drawdown_end.timestamp() * 1000,
+            })
+        except ValueError:
+            strat_stats.update({
+                'max_drawdown': 0.0,
+                'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc),
+                'drawdown_start_ts': 0,
+                'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc),
+                'drawdown_end_ts': 0,
+            })
+
     strategy_results = generate_strategy_metrics(stake_currency=stake_currency,
                                                  max_open_trades=max_open_trades,
                                                  all_results=all_results)
@@ -273,7 +352,7 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
 
     output = [[
         t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'],
-        t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_pct_total'],
+        t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_total_pct'],
     ] for t in sell_reason_stats]
     return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
 
@@ -298,6 +377,35 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
 
 
+def text_table_add_metrics(strat_results: Dict) -> str:
+    if len(strat_results['trades']) > 0:
+        min_trade = min(strat_results['trades'], key=lambda x: x['open_date'])
+        metrics = [
+            ('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Total trades', strat_results['total_trades']),
+            ('First trade', min_trade['open_date'].strftime(DATETIME_PRINT_FORMAT)),
+            ('First trade Pair', min_trade['pair']),
+            ('Total Profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"),
+            ('Trades per day', strat_results['trades_per_day']),
+            ('Best day', f"{round(strat_results['backtest_best_day'] * 100, 2)}%"),
+            ('Worst day', f"{round(strat_results['backtest_worst_day'] * 100, 2)}%"),
+            ('Days win/draw/lose', f"{strat_results['winning_days']} / "
+                f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
+            ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
+            ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
+            ('', ''),  # Empty line to improve readability
+            ('Max Drawdown', f"{round(strat_results['max_drawdown'] * 100, 2)}%"),
+            ('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)),
+            ('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"),
+        ]
+
+        return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl")
+    else:
+        return ''
+
+
 def show_backtest_results(config: Dict, backtest_stats: Dict):
     stake_currency = config['stake_currency']
 
@@ -312,15 +420,21 @@ def show_backtest_results(config: Dict, backtest_stats: Dict):
 
         table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
                                        stake_currency=stake_currency)
-        if isinstance(table, str):
+        if isinstance(table, str) and len(table) > 0:
             print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
         print(table)
 
         table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
-        if isinstance(table, str):
+        if isinstance(table, str) and len(table) > 0:
             print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
         print(table)
-        if isinstance(table, str):
+
+        table = text_table_add_metrics(results)
+        if isinstance(table, str) and len(table) > 0:
+            print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
+        print(table)
+
+        if isinstance(table, str) and len(table) > 0:
             print('=' * len(table.splitlines()[0]))
         print()
 

freqtrade/pairlist/AgeFilter.py

@@ -5,6 +5,7 @@ import logging
 import arrow
 from typing import Any, Dict
 
+from freqtrade.exceptions import OperationalException
 from freqtrade.misc import plural
 from freqtrade.pairlist.IPairList import IPairList
 
@@ -23,7 +24,13 @@ class AgeFilter(IPairList):
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
-        self._enabled = self._min_days_listed >= 1
+
+        if self._min_days_listed < 1:
+            raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
+        if self._min_days_listed > exchange.ohlcv_candle_limit:
+            raise OperationalException("AgeFilter requires min_days_listed to not exceed "
+                                       "exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit})")
 
     @property
     def needstickers(self) -> bool:
@@ -69,7 +76,7 @@ class AgeFilter(IPairList):
                 return True
             else:
                 self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
-                                                 f"because age is less than "
+                                                 f"because age {len(daily_candles)} is less than "
                                                  f"{self._min_days_listed} "
                                                  f"{plural(self._min_days_listed, 'day')}")
                 return False

freqtrade/pairlist/IPairList.py

@@ -162,6 +162,11 @@ class IPairList(ABC):
                                f"{self._exchange.name}. Removing it from whitelist..")
                 continue
 
+            if not self._exchange.market_is_tradable(markets[pair]):
+                logger.warning(f"Pair {pair} is not tradable with Freqtrade."
+                               "Removing it from whitelist..")
+                continue
+
             if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
                 logger.warning(f"Pair {pair} is not compatible with your stake currency "
                                f"{self._config['stake_currency']}. Removing it from whitelist..")

freqtrade/pairlist/PriceFilter.py

@@ -4,6 +4,7 @@ Price pair list filter
 import logging
 from typing import Any, Dict
 
+from freqtrade.exceptions import OperationalException
 from freqtrade.pairlist.IPairList import IPairList
 
 
@@ -18,7 +19,17 @@ class PriceFilter(IPairList):
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
         self._low_price_ratio = pairlistconfig.get('low_price_ratio', 0)
-        self._enabled = self._low_price_ratio != 0
+        if self._low_price_ratio < 0:
+            raise OperationalException("PriceFilter requires low_price_ratio to be >= 0")
+        self._min_price = pairlistconfig.get('min_price', 0)
+        if self._min_price < 0:
+            raise OperationalException("PriceFilter requires min_price to be >= 0")
+        self._max_price = pairlistconfig.get('max_price', 0)
+        if self._max_price < 0:
+            raise OperationalException("PriceFilter requires max_price to be >= 0")
+        self._enabled = ((self._low_price_ratio > 0) or
+                         (self._min_price > 0) or
+                         (self._max_price > 0))
 
     @property
     def needstickers(self) -> bool:
@@ -33,7 +44,18 @@ class PriceFilter(IPairList):
         """
         Short whitelist method description - used for startup-messages
         """
-        return f"{self.name} - Filtering pairs priced below {self._low_price_ratio * 100}%."
+        active_price_filters = []
+        if self._low_price_ratio != 0:
+            active_price_filters.append(f"below {self._low_price_ratio * 100}%")
+        if self._min_price != 0:
+            active_price_filters.append(f"below {self._min_price:.8f}")
+        if self._max_price != 0:
+            active_price_filters.append(f"above {self._max_price:.8f}")
+
+        if len(active_price_filters):
+            return f"{self.name} - Filtering pairs priced {' or '.join(active_price_filters)}."
+
+        return f"{self.name} - No price filters configured."
 
     def _validate_pair(self, ticker) -> bool:
         """
@@ -41,15 +63,33 @@ class PriceFilter(IPairList):
         :param ticker: ticker dict as returned from ccxt.load_markets()
         :return: True if the pair can stay, false if it should be removed
         """
-        if ticker['last'] is None:
+        if ticker['last'] is None or ticker['last'] == 0:
             self.log_on_refresh(logger.info,
                                 f"Removed {ticker['symbol']} from whitelist, because "
                                 "ticker['last'] is empty (Usually no trade in the last 24h).")
             return False
-        compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last'])
+
-        changeperc = compare / ticker['last']
+        # Perform low_price_ratio check.
-        if changeperc > self._low_price_ratio:
+        if self._low_price_ratio != 0:
-            self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
+            compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last'])
-                                             f"because 1 unit is {changeperc * 100:.3f}%")
+            changeperc = compare / ticker['last']
-            return False
+            if changeperc > self._low_price_ratio:
+                self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
+                                                 f"because 1 unit is {changeperc * 100:.3f}%")
+                return False
+
+        # Perform min_price check.
+        if self._min_price != 0:
+            if ticker['last'] < self._min_price:
+                self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
+                                                 f"because last price < {self._min_price:.8f}")
+                return False
+
+        # Perform max_price check.
+        if self._max_price != 0:
+            if ticker['last'] > self._max_price:
+                self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
+                                                 f"because last price > {self._max_price:.8f}")
+                return False
+
         return True

freqtrade/persistence.py

@@ -2,7 +2,7 @@
 This module contains the class to persist trades into SQLite
 """
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from decimal import Decimal
 from typing import Any, Dict, List, Optional
 
@@ -17,6 +17,7 @@ from sqlalchemy.orm.session import sessionmaker
 from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
+from freqtrade.misc import safe_value_fallback
 
 logger = logging.getLogger(__name__)
 
@@ -86,7 +87,7 @@ def check_migrate(engine) -> None:
         logger.debug(f'trying {table_back_name}')
 
     # Check for latest column
-    if not has_column(cols, 'timeframe'):
+    if not has_column(cols, 'amount_requested'):
         logger.info(f'Running database migration - backup available as {table_back_name}')
 
         fee_open = get_column_def(cols, 'fee_open', 'fee')
@@ -119,6 +120,7 @@ def check_migrate(engine) -> None:
             cols, 'close_profit_abs',
             f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
         sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
+        amount_requested = get_column_def(cols, 'amount_requested', 'amount')
 
         # Schema migration necessary
         engine.execute(f"alter table trades rename to {table_back_name}")
@@ -134,7 +136,7 @@ def check_migrate(engine) -> None:
                 fee_open, fee_open_cost, fee_open_currency,
                 fee_close, fee_close_cost, fee_open_currency, open_rate,
                 open_rate_requested, close_rate, close_rate_requested, close_profit,
-                stake_amount, amount, open_date, close_date, open_order_id,
+                stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
                 stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
                 stoploss_order_id, stoploss_last_update,
                 max_rate, min_rate, sell_reason, sell_order_status, strategy,
@@ -153,7 +155,7 @@ def check_migrate(engine) -> None:
                 {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
                 open_rate, {open_rate_requested} open_rate_requested, close_rate,
                 {close_rate_requested} close_rate_requested, close_profit,
-                stake_amount, amount, open_date, close_date, open_order_id,
+                stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
                 {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
                 {initial_stop_loss} initial_stop_loss,
                 {initial_stop_loss_pct} initial_stop_loss_pct,
@@ -215,6 +217,7 @@ class Trade(_DECL_BASE):
     close_profit_abs = Column(Float)
     stake_amount = Column(Float, nullable=False)
     amount = Column(Float)
+    amount_requested = Column(Float)
     open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
     close_date = Column(DateTime)
     open_order_id = Column(String)
@@ -256,6 +259,7 @@ class Trade(_DECL_BASE):
             'is_open': self.is_open,
             'exchange': self.exchange,
             'amount': round(self.amount, 8),
+            'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None,
             'stake_amount': round(self.stake_amount, 8),
             'strategy': self.strategy,
             'ticker_interval': self.timeframe,  # DEPRECATED
@@ -270,16 +274,17 @@ class Trade(_DECL_BASE):
 
             'open_date_hum': arrow.get(self.open_date).humanize(),
             'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"),
-            'open_timestamp': int(self.open_date.timestamp() * 1000),
+            'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
             'open_rate': self.open_rate,
             'open_rate_requested': self.open_rate_requested,
-            'open_trade_price': self.open_trade_price,
+            'open_trade_price': round(self.open_trade_price, 8),
 
             'close_date_hum': (arrow.get(self.close_date).humanize()
                                if self.close_date else None),
             'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S")
                            if self.close_date else None),
-            'close_timestamp': int(self.close_date.timestamp() * 1000) if self.close_date else None,
+            'close_timestamp': int(self.close_date.replace(
+                tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None,
             'close_rate': self.close_rate,
             'close_rate_requested': self.close_rate_requested,
             'close_profit': self.close_profit,
@@ -294,8 +299,8 @@ class Trade(_DECL_BASE):
             'stoploss_order_id': self.stoploss_order_id,
             'stoploss_last_update': (self.stoploss_last_update.strftime("%Y-%m-%d %H:%M:%S")
                                      if self.stoploss_last_update else None),
-            'stoploss_last_update_timestamp': (int(self.stoploss_last_update.timestamp() * 1000)
+            'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace(
-                                               if self.stoploss_last_update else None),
+                tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None,
             'initial_stop_loss': self.initial_stop_loss,  # Deprecated - should not be used
             'initial_stop_loss_abs': self.initial_stop_loss,
             'initial_stop_loss_ratio': (self.initial_stop_loss_pct
@@ -360,25 +365,25 @@ class Trade(_DECL_BASE):
     def update(self, order: Dict) -> None:
         """
         Updates this entity with amount and actual open/close rates.
-        :param order: order retrieved by exchange.get_order()
+        :param order: order retrieved by exchange.fetch_order()
         :return: None
         """
         order_type = order['type']
         # Ignore open and cancelled orders
-        if order['status'] == 'open' or order['price'] is None:
+        if order['status'] == 'open' or safe_value_fallback(order, 'average', 'price') is None:
             return
 
         logger.info('Updating trade (id=%s) ...', self.id)
 
         if order_type in ('market', 'limit') and order['side'] == 'buy':
             # Update open rate and actual amount
-            self.open_rate = Decimal(order['price'])
+            self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price'))
-            self.amount = Decimal(order.get('filled', order['amount']))
+            self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount'))
             self.recalc_open_trade_price()
             logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self)
             self.open_order_id = None
         elif order_type in ('market', 'limit') and order['side'] == 'sell':
-            self.close(order['price'])
+            self.close(safe_value_fallback(order, 'average', 'price'))
             logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self)
         elif order_type in ('stop_loss_limit', 'stop-loss', 'stop'):
             self.stoploss_order_id = None

freqtrade/plot/plotting.py

@@ -8,13 +8,16 @@ from freqtrade.configuration import TimeRange
 from freqtrade.data.btanalysis import (calculate_max_drawdown,
                                        combine_dataframes_with_mean,
                                        create_cum_profit,
-                                       extract_trades_of_period, load_trades)
+                                       extract_trades_of_period,
+                                       load_trades)
 from freqtrade.data.converter import trim_dataframe
+from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history import load_data
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_prev_date
 from freqtrade.misc import pair_to_filename
-from freqtrade.resolvers import StrategyResolver
+from freqtrade.resolvers import ExchangeResolver, StrategyResolver
+from freqtrade.strategy import IStrategy
 
 logger = logging.getLogger(__name__)
 
@@ -35,15 +38,15 @@ def init_plotscript(config):
     """
 
     if "pairs" in config:
-        pairs = config["pairs"]
+        pairs = config['pairs']
     else:
-        pairs = config["exchange"]["pair_whitelist"]
+        pairs = config['exchange']['pair_whitelist']
 
     # Set timerange to use
-    timerange = TimeRange.parse_timerange(config.get("timerange"))
+    timerange = TimeRange.parse_timerange(config.get('timerange'))
 
     data = load_data(
-        datadir=config.get("datadir"),
+        datadir=config.get('datadir'),
         pairs=pairs,
         timeframe=config.get('timeframe', '5m'),
         timerange=timerange,
@@ -51,19 +54,22 @@ def init_plotscript(config):
     )
 
     no_trades = False
+    filename = config.get('exportfilename')
     if config.get('no_trades', False):
         no_trades = True
-    elif not config['exportfilename'].is_file() and config['trade_source'] == 'file':
+    elif config['trade_source'] == 'file':
-        logger.warning("Backtest file is missing skipping trades.")
+        if not filename.is_dir() and not filename.is_file():
-        no_trades = True
+            logger.warning("Backtest file is missing skipping trades.")
+            no_trades = True
 
     trades = load_trades(
         config['trade_source'],
         db_url=config.get('db_url'),
-        exportfilename=config.get('exportfilename'),
+        exportfilename=filename,
-        no_trades=no_trades
+        no_trades=no_trades,
+        strategy=config.get('strategy'),
     )
-    trades = trim_dataframe(trades, timerange, 'open_time')
+    trades = trim_dataframe(trades, timerange, 'open_date')
 
     return {"ohlcv": data,
             "trades": trades,
@@ -163,10 +169,11 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
     if trades is not None and len(trades) > 0:
         # Create description for sell summarizing the trade
         trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, "
-                                                  f"{row['sell_reason']}, {row['duration']} min",
+                                                  f"{row['sell_reason']}, "
+                                                  f"{row['trade_duration']} min",
                                                   axis=1)
         trade_buys = go.Scatter(
-            x=trades["open_time"],
+            x=trades["open_date"],
             y=trades["open_rate"],
             mode='markers',
             name='Trade buy',
@@ -181,7 +188,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
         )
 
         trade_sells = go.Scatter(
-            x=trades.loc[trades['profit_percent'] > 0, "close_time"],
+            x=trades.loc[trades['profit_percent'] > 0, "close_date"],
             y=trades.loc[trades['profit_percent'] > 0, "close_rate"],
             text=trades.loc[trades['profit_percent'] > 0, "desc"],
             mode='markers',
@@ -194,7 +201,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
             )
         )
         trade_sells_loss = go.Scatter(
-            x=trades.loc[trades['profit_percent'] <= 0, "close_time"],
+            x=trades.loc[trades['profit_percent'] <= 0, "close_date"],
             y=trades.loc[trades['profit_percent'] <= 0, "close_rate"],
             text=trades.loc[trades['profit_percent'] <= 0, "desc"],
             mode='markers',
@@ -467,6 +474,8 @@ def load_and_plot_trades(config: Dict[str, Any]):
     """
     strategy = StrategyResolver.load_strategy(config)
 
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
+    IStrategy.dp = DataProvider(config, exchange)
     plot_elements = init_plotscript(config)
     trades = plot_elements['trades']
     pair_counter = 0
@@ -482,13 +491,13 @@ def load_and_plot_trades(config: Dict[str, Any]):
             pair=pair,
             data=df_analyzed,
             trades=trades_pair,
-            indicators1=config.get("indicators1", []),
+            indicators1=config.get('indicators1', []),
-            indicators2=config.get("indicators2", []),
+            indicators2=config.get('indicators2', []),
             plot_config=strategy.plot_config if hasattr(strategy, 'plot_config') else {}
         )
 
         store_plot_file(fig, filename=generate_plot_filename(pair, config['timeframe']),
-                        directory=config['user_data_dir'] / "plot")
+                        directory=config['user_data_dir'] / 'plot')
 
     logger.info('End of plotting process. %s plots generated', pair_counter)
 
@@ -505,8 +514,8 @@ def plot_profit(config: Dict[str, Any]) -> None:
     # Filter trades to relevant pairs
     # Remove open pairs - we don't know the profit yet so can't calculate profit for these.
     # Also, If only one open pair is left, then the profit-generation would fail.
-    trades = trades[(trades['pair'].isin(plot_elements["pairs"]))
+    trades = trades[(trades['pair'].isin(plot_elements['pairs']))
-                    & (~trades['close_time'].isnull())
+                    & (~trades['close_date'].isnull())
                     ]
     if len(trades) == 0:
         raise OperationalException("No trades found, cannot generate Profit-plot without "
@@ -514,7 +523,7 @@ def plot_profit(config: Dict[str, Any]) -> None:
 
     # Create an average close price of all the pairs that were involved.
     # this could be useful to gauge the overall market trend
-    fig = generate_profit_graph(plot_elements["pairs"], plot_elements["ohlcv"],
+    fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'],
                                 trades, config.get('timeframe', '5m'))
     store_plot_file(fig, filename='freqtrade-profit-plot.html',
-                    directory=config['user_data_dir'] / "plot", auto_open=True)
+                    directory=config['user_data_dir'] / 'plot', auto_open=True)

freqtrade/resolvers/hyperopt_resolver.py

@@ -23,7 +23,7 @@ class HyperOptResolver(IResolver):
     object_type = IHyperOpt
     object_type_str = "Hyperopt"
     user_subdir = USERPATH_HYPEROPTS
-    initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve()
+    initial_search_path = None
 
     @staticmethod
     def load_hyperopt(config: Dict) -> IHyperOpt:
@@ -42,14 +42,14 @@ class HyperOptResolver(IResolver):
                                                 extra_dir=config.get('hyperopt_path'))
 
         if not hasattr(hyperopt, 'populate_indicators'):
-            logger.warning("Hyperopt class does not provide populate_indicators() method. "
+            logger.info("Hyperopt class does not provide populate_indicators() method. "
-                           "Using populate_indicators from the strategy.")
+                        "Using populate_indicators from the strategy.")
         if not hasattr(hyperopt, 'populate_buy_trend'):
-            logger.warning("Hyperopt class does not provide populate_buy_trend() method. "
+            logger.info("Hyperopt class does not provide populate_buy_trend() method. "
-                           "Using populate_buy_trend from the strategy.")
+                        "Using populate_buy_trend from the strategy.")
         if not hasattr(hyperopt, 'populate_sell_trend'):
-            logger.warning("Hyperopt class does not provide populate_sell_trend() method. "
+            logger.info("Hyperopt class does not provide populate_sell_trend() method. "
-                           "Using populate_sell_trend from the strategy.")
+                        "Using populate_sell_trend from the strategy.")
         return hyperopt
 
 

freqtrade/rpc/api_server.py

@@ -16,7 +16,9 @@ from werkzeug.security import safe_str_cmp
 from werkzeug.serving import make_server
 
 from freqtrade.__init__ import __version__
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.rpc.rpc import RPC, RPCException
+from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
 
 logger = logging.getLogger(__name__)
 
@@ -31,7 +33,7 @@ class ArrowJSONEncoder(JSONEncoder):
             elif isinstance(obj, date):
                 return obj.strftime("%Y-%m-%d")
             elif isinstance(obj, datetime):
-                return obj.strftime("%Y-%m-%d %H:%M:%S")
+                return obj.strftime(DATETIME_PRINT_FORMAT)
             iterable = iter(obj)
         except TypeError:
             pass
@@ -55,7 +57,7 @@ def require_login(func: Callable[[Any, Any], Any]):
 
 
 # Type should really be Callable[[ApiServer], Any], but that will create a circular dependency
-def rpc_catch_errors(func: Callable[[Any], Any]):
+def rpc_catch_errors(func: Callable[..., Any]):
 
     def func_wrapper(obj, *args, **kwargs):
 
@@ -105,6 +107,9 @@ class ApiServer(RPC):
         # Register application handling
         self.register_rest_rpc_urls()
 
+        if self._config.get('fiat_display_currency', None):
+            self._fiat_converter = CryptoToFiatConverter()
+
         thread = threading.Thread(target=self.run, daemon=True)
         thread.start()
 
@@ -182,6 +187,7 @@ class ApiServer(RPC):
         self.app.add_url_rule(f'{BASE_URI}/count', 'count', view_func=self._count, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/daily', 'daily', view_func=self._daily, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/edge', 'edge', view_func=self._edge, methods=['GET'])
+        self.app.add_url_rule(f'{BASE_URI}/logs', 'log', view_func=self._get_logs, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/profit', 'profit',
                               view_func=self._profit, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/performance', 'performance',
@@ -196,6 +202,8 @@ class ApiServer(RPC):
                               view_func=self._ping, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/trades', 'trades',
                               view_func=self._trades, methods=['GET'])
+        self.app.add_url_rule(f'{BASE_URI}/trades/<int:tradeid>', 'trades_delete',
+                              view_func=self._trades_delete, methods=['DELETE'])
         # Combined actions and infos
         self.app.add_url_rule(f'{BASE_URI}/blacklist', 'blacklist', view_func=self._blacklist,
                               methods=['GET', 'POST'])
@@ -342,6 +350,18 @@ class ApiServer(RPC):
 
         return self.rest_dump(stats)
 
+    @require_login
+    @rpc_catch_errors
+    def _get_logs(self):
+        """
+        Returns latest logs
+         get:
+          param:
+            limit: Only get a certain number of records
+        """
+        limit = int(request.args.get('limit', 0)) or None
+        return self.rest_dump(self._rpc_get_logs(limit))
+
     @require_login
     @rpc_catch_errors
     def _edge(self):
@@ -420,6 +440,19 @@ class ApiServer(RPC):
         results = self._rpc_trade_history(limit)
         return self.rest_dump(results)
 
+    @require_login
+    @rpc_catch_errors
+    def _trades_delete(self, tradeid):
+        """
+        Handler for DELETE /trades/<tradeid> endpoint.
+        Removes the trade from the database (tries to cancel open orders first!)
+        get:
+          param:
+            tradeid: Numeric trade-id assigned to the trade.
+        """
+        result = self._rpc_delete(tradeid)
+        return self.rest_dump(result)
+
     @require_login
     @rpc_catch_errors
     def _whitelist(self):

freqtrade/rpc/rpc.py

@@ -6,12 +6,14 @@ from abc import abstractmethod
 from datetime import date, datetime, timedelta
 from enum import Enum
 from math import isnan
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import arrow
 from numpy import NAN, mean
 
-from freqtrade.exceptions import DependencyException, TemporaryError
+from freqtrade.exceptions import ExchangeError, PricingError
+from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
+from freqtrade.loggers import bufferHandler
 from freqtrade.misc import shorten_date
 from freqtrade.persistence import Trade
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -103,6 +105,8 @@ class RPC:
             'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached'),
             'ticker_interval': config['timeframe'],  # DEPRECATED
             'timeframe': config['timeframe'],
+            'timeframe_ms': timeframe_to_msecs(config['timeframe']),
+            'timeframe_min': timeframe_to_minutes(config['timeframe']),
             'exchange': config['exchange']['name'],
             'strategy': config['strategy'],
             'forcebuy_enabled': config.get('forcebuy_enable', False),
@@ -126,11 +130,11 @@ class RPC:
             for trade in trades:
                 order = None
                 if trade.open_order_id:
-                    order = self._freqtrade.exchange.get_order(trade.open_order_id, trade.pair)
+                    order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
                 # calculate profit and send message to user
                 try:
                     current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
-                except DependencyException:
+                except (ExchangeError, PricingError):
                     current_rate = NAN
                 current_profit = trade.calc_profit_ratio(current_rate)
                 current_profit_abs = trade.calc_profit(current_rate)
@@ -154,6 +158,7 @@ class RPC:
                     current_profit_abs=current_profit_abs,
                     stoploss_current_dist=stoploss_current_dist,
                     stoploss_current_dist_ratio=round(stoploss_current_dist_ratio, 8),
+                    stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2),
                     stoploss_entry_dist=stoploss_entry_dist,
                     stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8),
                     open_order='({} {} rem={:.8f})'.format(
@@ -174,7 +179,7 @@ class RPC:
                 # calculate profit and send message to user
                 try:
                     current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
-                except DependencyException:
+                except (PricingError, ExchangeError):
                     current_rate = NAN
                 trade_percent = (100 * trade.calc_profit_ratio(current_rate))
                 trade_profit = trade.calc_profit(current_rate)
@@ -220,22 +225,20 @@ class RPC:
             ]).order_by(Trade.close_date).all()
             curdayprofit = sum(trade.close_profit_abs for trade in trades)
             profit_days[profitday] = {
-                'amount': f'{curdayprofit:.8f}',
+                'amount': curdayprofit,
                 'trades': len(trades)
             }
 
         data = [
             {
                 'date': key,
-                'abs_profit': f'{float(value["amount"]):.8f}',
+                'abs_profit': value["amount"],
-                'fiat_value': '{value:.3f}'.format(
+                'fiat_value': self._fiat_converter.convert_amount(
-                    value=self._fiat_converter.convert_amount(
                         value['amount'],
                         stake_currency,
                         fiat_display_currency
                     ) if self._fiat_converter else 0,
-                ),
+                'trade_count': value["trades"],
-                'trade_count': f'{value["trades"]}',
             }
             for key, value in profit_days.items()
         ]
@@ -248,9 +251,10 @@ class RPC:
     def _rpc_trade_history(self, limit: int) -> Dict:
         """ Returns the X last trades """
         if limit > 0:
-            trades = Trade.get_trades().order_by(Trade.id.desc()).limit(limit)
+            trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
+                Trade.id.desc()).limit(limit)
         else:
-            trades = Trade.get_trades().order_by(Trade.id.desc()).all()
+            trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(Trade.id.desc()).all()
 
         output = [trade.to_json() for trade in trades]
 
@@ -269,6 +273,8 @@ class RPC:
         profit_closed_coin = []
         profit_closed_ratio = []
         durations = []
+        winning_trades = 0
+        losing_trades = 0
 
         for trade in trades:
             current_rate: float = 0.0
@@ -282,11 +288,15 @@ class RPC:
                 profit_ratio = trade.close_profit
                 profit_closed_coin.append(trade.close_profit_abs)
                 profit_closed_ratio.append(profit_ratio)
+                if trade.close_profit >= 0:
+                    winning_trades += 1
+                else:
+                    losing_trades += 1
             else:
                 # Get current rate
                 try:
                     current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
-                except DependencyException:
+                except (PricingError, ExchangeError):
                     current_rate = NAN
                 profit_ratio = trade.calc_profit_ratio(rate=current_rate)
 
@@ -344,6 +354,8 @@ class RPC:
             'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0],
             'best_pair': best_pair[0] if best_pair else '',
             'best_rate': round(best_pair[1] * 100, 2) if best_pair else 0,
+            'winning_trades': winning_trades,
+            'losing_trades': losing_trades,
         }
 
     def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict:
@@ -352,7 +364,7 @@ class RPC:
         total = 0.0
         try:
             tickers = self._freqtrade.exchange.get_tickers()
-        except (TemporaryError, DependencyException):
+        except (ExchangeError):
             raise RPCException('Error getting current tickers.')
 
         self._freqtrade.wallets.update(require_update=False)
@@ -373,7 +385,7 @@ class RPC:
                         if pair.startswith(stake_currency):
                             rate = 1.0 / rate
                         est_stake = rate * balance.total
-                except (TemporaryError, DependencyException):
+                except (ExchangeError):
                     logger.warning(f" Could not get rate for pair {coin}.")
                     continue
             total = total + (est_stake or 0)
@@ -442,7 +454,7 @@ class RPC:
         def _exec_forcesell(trade: Trade) -> None:
             # Check if there is there is an open order
             if trade.open_order_id:
-                order = self._freqtrade.exchange.get_order(trade.open_order_id, trade.pair)
+                order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
 
                 # Cancel open LIMIT_BUY orders and close trade
                 if order and order['status'] == 'open' \
@@ -511,7 +523,7 @@ class RPC:
         # check if valid pair
 
         # check if pair already has an open pair
-        trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first()
+        trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
         if trade:
             raise RPCException(f'position for {pair} already open - id: {trade.id}')
 
@@ -520,11 +532,51 @@ class RPC:
 
         # execute buy
         if self._freqtrade.execute_buy(pair, stakeamount, price):
-            trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first()
+            trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
             return trade
         else:
             return None
 
+    def _rpc_delete(self, trade_id: str) -> Dict[str, Union[str, int]]:
+        """
+        Handler for delete <id>.
+        Delete the given trade and close eventually existing open orders.
+        """
+        with self._freqtrade._sell_lock:
+            c_count = 0
+            trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first()
+            if not trade:
+                logger.warning('delete trade: Invalid argument received')
+                raise RPCException('invalid argument')
+
+            # Try cancelling regular order if that exists
+            if trade.open_order_id:
+                try:
+                    self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair)
+                    c_count += 1
+                except (ExchangeError):
+                    pass
+
+            # cancel stoploss on exchange ...
+            if (self._freqtrade.strategy.order_types.get('stoploss_on_exchange')
+                    and trade.stoploss_order_id):
+                try:
+                    self._freqtrade.exchange.cancel_stoploss_order(trade.stoploss_order_id,
+                                                                   trade.pair)
+                    c_count += 1
+                except (ExchangeError):
+                    pass
+
+            Trade.session.delete(trade)
+            Trade.session.flush()
+            self._freqtrade.wallets.update()
+            return {
+                'result': 'success',
+                'trade_id': trade_id,
+                'result_msg': f'Deleted trade {trade_id}. Closed {c_count} open orders.',
+                'cancel_order_count': c_count,
+            }
+
     def _rpc_performance(self) -> List[Dict[str, Any]]:
         """
         Handler for performance.
@@ -580,6 +632,24 @@ class RPC:
                }
         return res
 
+    def _rpc_get_logs(self, limit: Optional[int]) -> Dict[str, Any]:
+        """Returns the last X logs"""
+        if limit:
+            buffer = bufferHandler.buffer[-limit:]
+        else:
+            buffer = bufferHandler.buffer
+        records = [[datetime.fromtimestamp(r.created).strftime("%Y-%m-%d %H:%M:%S"),
+                   r.created * 1000, r.name, r.levelname,
+                   r.message + ('\n' + r.exc_text if r.exc_text else '')]
+                   for r in buffer]
+
+        # Log format:
+        # [logtime-formatted, logepoch, logger-name, loglevel, message \n + exception]
+        # e.g. ["2020-08-27 11:35:01", 1598520901097.9397,
+        #       "freqtrade.worker", "INFO", "Starting worker develop"]
+
+        return {'log_count': len(records), 'logs': records}
+
     def _rpc_edge(self) -> List[Dict[str, Any]]:
         """ Returns information related to Edge """
         if not self._freqtrade.edge:

freqtrade/rpc/telegram.py

@@ -5,12 +5,14 @@ This module manage Telegram communication
 """
 import json
 import logging
+import arrow
 from typing import Any, Callable, Dict
 
 from tabulate import tabulate
 from telegram import ParseMode, ReplyKeyboardMarkup, Update
 from telegram.error import NetworkError, TelegramError
 from telegram.ext import CallbackContext, CommandHandler, Updater
+from telegram.utils.helpers import escape_markdown
 
 from freqtrade.__init__ import __version__
 from freqtrade.rpc import RPC, RPCException, RPCMessageType
@@ -92,6 +94,8 @@ class Telegram(RPC):
             CommandHandler('stop', self._stop),
             CommandHandler('forcesell', self._forcesell),
             CommandHandler('forcebuy', self._forcebuy),
+            CommandHandler('trades', self._trades),
+            CommandHandler('delete', self._delete_trade),
             CommandHandler('performance', self._performance),
             CommandHandler('daily', self._daily),
             CommandHandler('count', self._count),
@@ -100,6 +104,7 @@ class Telegram(RPC):
             CommandHandler('stopbuy', self._stopbuy),
             CommandHandler('whitelist', self._whitelist),
             CommandHandler('blacklist', self._blacklist),
+            CommandHandler('logs', self._logs),
             CommandHandler('edge', self._edge),
             CommandHandler('help', self._help),
             CommandHandler('version', self._version),
@@ -236,17 +241,18 @@ class Telegram(RPC):
                     ("*Close Profit:* `{close_profit_pct}`"
                      if r['close_profit_pct'] is not None else ""),
                     "*Current Profit:* `{current_profit_pct:.2f}%`",
-
-                    # Adding initial stoploss only if it is different from stoploss
-                    "*Initial Stoploss:* `{initial_stop_loss:.8f}` " +
-                    ("`({initial_stop_loss_pct:.2f}%)`") if (
-                            r['stop_loss'] != r['initial_stop_loss']
-                            and r['initial_stop_loss_pct'] is not None) else "",
-
-                    # Adding stoploss and stoploss percentage only if it is not None
-                    "*Stoploss:* `{stop_loss:.8f}` " +
-                    ("`({stop_loss_pct:.2f}%)`" if r['stop_loss_pct'] else ""),
                 ]
+                if (r['stop_loss'] != r['initial_stop_loss']
+                        and r['initial_stop_loss_pct'] is not None):
+                    # Adding initial stoploss only if it is different from stoploss
+                    lines.append("*Initial Stoploss:* `{initial_stop_loss:.8f}` "
+                                 "`({initial_stop_loss_pct:.2f}%)`")
+
+                # Adding stoploss and stoploss percentage only if it is not None
+                lines.append("*Stoploss:* `{stop_loss:.8f}` " +
+                             ("`({stop_loss_pct:.2f}%)`" if r['stop_loss_pct'] else ""))
+                lines.append("*Stoploss distance:* `{stoploss_current_dist:.8f}` "
+                             "`({stoploss_current_dist_pct:.2f}%)`")
                 if r['open_order']:
                     if r['sell_order_status']:
                         lines.append("*Open Order:* `{open_order}` - `{sell_order_status}`")
@@ -302,8 +308,8 @@ class Telegram(RPC):
             )
             stats_tab = tabulate(
                 [[day['date'],
-                  f"{day['abs_profit']} {stats['stake_currency']}",
+                  f"{day['abs_profit']:.8f} {stats['stake_currency']}",
-                  f"{day['fiat_value']} {stats['fiat_display_currency']}",
+                  f"{day['fiat_value']:.3f} {stats['fiat_display_currency']}",
                   f"{day['trade_count']} trades"] for day in stats['data']],
                 headers=[
                     'Day',
@@ -366,7 +372,9 @@ class Telegram(RPC):
                              f"∙ `{profit_all_fiat:.3f} {fiat_disp_cur}`\n"
                              f"*Total Trade Count:* `{trade_count}`\n"
                              f"*First Trade opened:* `{first_trade_date}`\n"
-                             f"*Latest Trade opened:* `{latest_trade_date}`")
+                             f"*Latest Trade opened:* `{latest_trade_date}\n`"
+                             f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`"
+                             )
             if stats['closed_trade_count'] > 0:
                 markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n"
                                  f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`")
@@ -494,6 +502,62 @@ class Telegram(RPC):
         except RPCException as e:
             self._send_msg(str(e))
 
+    @authorized_only
+    def _trades(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /trades <n>
+        Returns last n recent trades.
+        :param bot: telegram bot
+        :param update: message update
+        :return: None
+        """
+        stake_cur = self._config['stake_currency']
+        try:
+            nrecent = int(context.args[0])
+        except (TypeError, ValueError, IndexError):
+            nrecent = 10
+        try:
+            trades = self._rpc_trade_history(
+                nrecent
+            )
+            trades_tab = tabulate(
+                [[arrow.get(trade['open_date']).humanize(),
+                  trade['pair'],
+                  f"{(100 * trade['close_profit']):.2f}% ({trade['close_profit_abs']})"]
+                 for trade in trades['trades']],
+                headers=[
+                    'Open Date',
+                    'Pair',
+                    f'Profit ({stake_cur})',
+                ],
+                tablefmt='simple')
+            message = (f"<b>{min(trades['trades_count'], nrecent)} recent trades</b>:\n"
+                       + (f"<pre>{trades_tab}</pre>" if trades['trades_count'] > 0 else ''))
+            self._send_msg(message, parse_mode=ParseMode.HTML)
+        except RPCException as e:
+            self._send_msg(str(e))
+
+    @authorized_only
+    def _delete_trade(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /delete <id>.
+        Delete the given trade
+        :param bot: telegram bot
+        :param update: message update
+        :return: None
+        """
+
+        trade_id = context.args[0] if len(context.args) > 0 else None
+        try:
+            msg = self._rpc_delete(trade_id)
+            self._send_msg((
+                '`{result_msg}`\n'
+                'Please make sure to take care of this asset on the exchange manually.'
+            ).format(**msg))
+
+        except RPCException as e:
+            self._send_msg(str(e))
+
     @authorized_only
     def _performance(self, update: Update, context: CallbackContext) -> None:
         """
@@ -576,6 +640,38 @@ class Telegram(RPC):
         except RPCException as e:
             self._send_msg(str(e))
 
+    @authorized_only
+    def _logs(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /logs
+        Shows the latest logs
+        """
+        try:
+            try:
+                limit = int(context.args[0])
+            except (TypeError, ValueError, IndexError):
+                limit = 10
+            logs = self._rpc_get_logs(limit)['logs']
+            msgs = ''
+            msg_template = "*{}* {}: {} \\- `{}`"
+            for logrec in logs:
+                msg = msg_template.format(escape_markdown(logrec[0], version=2),
+                                          escape_markdown(logrec[2], version=2),
+                                          escape_markdown(logrec[3], version=2),
+                                          escape_markdown(logrec[4], version=2))
+                if len(msgs + msg) + 10 >= MAX_TELEGRAM_MESSAGE_LENGTH:
+                    # Send message immediately if it would become too long
+                    self._send_msg(msgs, parse_mode=ParseMode.MARKDOWN_V2)
+                    msgs = msg + '\n'
+                else:
+                    # Append message to messages to send
+                    msgs += msg + '\n'
+
+            if msgs:
+                self._send_msg(msgs, parse_mode=ParseMode.MARKDOWN_V2)
+        except RPCException as e:
+            self._send_msg(str(e))
+
     @authorized_only
     def _edge(self, update: Update, context: CallbackContext) -> None:
         """
@@ -607,10 +703,12 @@ class Telegram(RPC):
                    "         *table :* `will display trades in a table`\n"
                    "                `pending buy orders are marked with an asterisk (*)`\n"
                    "                `pending sell orders are marked with a double asterisk (**)`\n"
+                   "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n"
                    "*/profit:* `Lists cumulative profit from all finished trades`\n"
                    "*/forcesell <trade_id>|all:* `Instantly sells the given trade or all trades, "
                    "regardless of profit`\n"
                    f"{forcebuy_text if self._config.get('forcebuy_enable', False) else ''}"
+                   "*/delete <trade_id>:* `Instantly delete the given trade in the database`\n"
                    "*/performance:* `Show performance of each finished trade grouped by pair`\n"
                    "*/daily <n>:* `Shows profit or loss per day, over the last n days`\n"
                    "*/count:* `Show number of trades running compared to allowed number of trades`"
@@ -619,6 +717,7 @@ class Telegram(RPC):
                    "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n"
                    "*/reload_config:* `Reload configuration file` \n"
                    "*/show_config:* `Show running configuration` \n"
+                   "*/logs [limit]:* `Show latest logs - defaults to 10` \n"
                    "*/whitelist:* `Show current whitelist` \n"
                    "*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs "
                    "to the blacklist.` \n"

freqtrade/strategy/interface.py

@@ -7,20 +7,20 @@ import warnings
 from abc import ABC, abstractmethod
 from datetime import datetime, timezone
 from enum import Enum
-from typing import Dict, NamedTuple, Optional, Tuple
+from typing import Dict, List, NamedTuple, Optional, Tuple
 
 import arrow
 from pandas import DataFrame
 
+from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.exceptions import StrategyError
+from freqtrade.exceptions import OperationalException, StrategyError
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.exchange.exchange import timeframe_to_next_date
 from freqtrade.persistence import Trade
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
-from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.wallets import Wallets
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -45,6 +45,10 @@ class SellType(Enum):
     EMERGENCY_SELL = "emergency_sell"
     NONE = ""
 
+    def __str__(self):
+        # explicitly convert to String to help with exporting data.
+        return self.value
+
 
 class SellCheckTuple(NamedTuple):
     """
@@ -191,6 +195,63 @@ class IStrategy(ABC):
         """
         return False
 
+    def bot_loop_start(self, **kwargs) -> None:
+        """
+        Called at the start of the bot iteration (one loop).
+        Might be used to perform pair-independent tasks
+        (e.g. gather some remote resource for comparison)
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        """
+        pass
+
+    def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float,
+                            time_in_force: str, **kwargs) -> bool:
+        """
+        Called right before placing a buy order.
+        Timing for this function is critical, so avoid doing heavy computations or
+        network requests in this method.
+
+        For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
+
+        When not implemented by a strategy, returns True (always confirming).
+
+        :param pair: Pair that's about to be bought.
+        :param order_type: Order type (as configured in order_types). usually limit or market.
+        :param amount: Amount in target (quote) currency that's going to be traded.
+        :param rate: Rate that's going to be used when using limit orders
+        :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        :return bool: When True is returned, then the buy-order is placed on the exchange.
+            False aborts the process
+        """
+        return True
+
+    def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float,
+                           rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool:
+        """
+        Called right before placing a regular sell order.
+        Timing for this function is critical, so avoid doing heavy computations or
+        network requests in this method.
+
+        For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
+
+        When not implemented by a strategy, returns True (always confirming).
+
+        :param pair: Pair that's about to be sold.
+        :param trade: trade object.
+        :param order_type: Order type (as configured in order_types). usually limit or market.
+        :param amount: Amount in quote currency.
+        :param rate: Rate that's going to be used when using limit orders
+        :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
+        :param sell_reason: Sell reason.
+            Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss',
+                           'sell_signal', 'force_sell', 'emergency_sell']
+        :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+        :return bool: When True is returned, then the sell-order is placed on the exchange.
+            False aborts the process
+        """
+        return True
+
     def informative_pairs(self) -> ListPairsWithTimeframes:
         """
         Define additional, informative pair/interval combinations to be cached from the exchange.
@@ -204,6 +265,10 @@ class IStrategy(ABC):
         """
         return []
 
+###
+# END - Intended to be overridden by strategy
+###
+
     def get_strategy_name(self) -> str:
         """
         Returns strategy class name
@@ -233,13 +298,25 @@ class IStrategy(ABC):
         if pair in self._pair_locked_until:
             del self._pair_locked_until[pair]
 
-    def is_pair_locked(self, pair: str) -> bool:
+    def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool:
         """
         Checks if a pair is currently locked
+        The 2nd, optional parameter ensures that locks are applied until the new candle arrives,
+        and not stop at 14:00:00 - while the next candle arrives at 14:00:02 leaving a gap
+        of 2 seconds for a buy to happen on an old signal.
+        :param: pair: "Pair to check"
+        :param candle_date: Date of the last candle. Optional, defaults to current date
+        :returns: locking state of the pair in question.
         """
         if pair not in self._pair_locked_until:
             return False
-        return self._pair_locked_until[pair] >= datetime.now(timezone.utc)
+        if not candle_date:
+            return self._pair_locked_until[pair] >= datetime.now(timezone.utc)
+        else:
+            # Locking should happen until a new candle arrives
+            lock_time = timeframe_to_next_date(self.timeframe, candle_date)
+            # lock_time = candle_date + timedelta(minutes=timeframe_to_minutes(self.timeframe))
+            return self._pair_locked_until[pair] > lock_time
 
     def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
@@ -273,6 +350,8 @@ class IStrategy(ABC):
             # Defs that only make change on new candle data.
             dataframe = self.analyze_ticker(dataframe, metadata)
             self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date']
+            if self.dp:
+                self.dp._set_cached_df(pair, self.timeframe, dataframe)
         else:
             logger.debug("Skipping TA Analysis for already analyzed candle")
             dataframe['buy'] = 0
@@ -284,13 +363,53 @@ class IStrategy(ABC):
 
         return dataframe
 
+    def analyze_pair(self, pair: str) -> None:
+        """
+        Fetch data for this pair from dataprovider and analyze.
+        Stores the dataframe into the dataprovider.
+        The analyzed dataframe is then accessible via `dp.get_analyzed_dataframe()`.
+        :param pair: Pair to analyze.
+        """
+        if not self.dp:
+            raise OperationalException("DataProvider not found.")
+        dataframe = self.dp.ohlcv(pair, self.timeframe)
+        if not isinstance(dataframe, DataFrame) or dataframe.empty:
+            logger.warning('Empty candle (OHLCV) data for pair %s', pair)
+            return
+
+        try:
+            df_len, df_close, df_date = self.preserve_df(dataframe)
+
+            dataframe = strategy_safe_wrapper(
+                self._analyze_ticker_internal, message=""
+            )(dataframe, {'pair': pair})
+
+            self.assert_df(dataframe, df_len, df_close, df_date)
+        except StrategyError as error:
+            logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}")
+            return
+
+        if dataframe.empty:
+            logger.warning('Empty dataframe for pair %s', pair)
+            return
+
+    def analyze(self, pairs: List[str]) -> None:
+        """
+        Analyze all pairs using analyze_pair().
+        :param pairs: List of pairs to analyze
+        """
+        for pair in pairs:
+            self.analyze_pair(pair)
+
     @staticmethod
     def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]:
         """ keep some data for dataframes """
         return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1]
 
     def assert_df(self, dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime):
-        """ make sure data is unmodified """
+        """
+        Ensure dataframe (length, last candle) was not modified, and has all elements we need.
+        """
         message = ""
         if df_len != len(dataframe):
             message = "length"
@@ -304,31 +423,17 @@ class IStrategy(ABC):
             else:
                 raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.")
 
-    def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]:
+    def get_signal(self, pair: str, timeframe: str, dataframe: DataFrame) -> Tuple[bool, bool]:
         """
-        Calculates current signal based several technical analysis indicators
+        Calculates current signal based based on the buy / sell columns of the dataframe.
+        Used by Bot to get the signal to buy or sell
         :param pair: pair in format ANT/BTC
-        :param interval: Interval to use (in min)
+        :param timeframe: timeframe to use
-        :param dataframe: Dataframe to analyze
+        :param dataframe: Analyzed dataframe to get signal from.
         :return: (Buy, Sell) A bool-tuple indicating buy/sell signal
         """
         if not isinstance(dataframe, DataFrame) or dataframe.empty:
-            logger.warning('Empty candle (OHLCV) data for pair %s', pair)
+            logger.warning(f'Empty candle (OHLCV) data for pair {pair}')
-            return False, False
-
-        try:
-            df_len, df_close, df_date = self.preserve_df(dataframe)
-            dataframe = strategy_safe_wrapper(
-                self._analyze_ticker_internal, message=""
-                )(dataframe, {'pair': pair})
-            self.assert_df(dataframe, df_len, df_close, df_date)
-        except StrategyError as error:
-            logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}")
-
-            return False, False
-
-        if dataframe.empty:
-            logger.warning('Empty dataframe for pair %s', pair)
             return False, False
 
         latest_date = dataframe['date'].max()
@@ -337,24 +442,18 @@ class IStrategy(ABC):
         latest_date = arrow.get(latest_date)
 
         # Check if dataframe is out of date
-        interval_minutes = timeframe_to_minutes(interval)
+        timeframe_minutes = timeframe_to_minutes(timeframe)
         offset = self.config.get('exchange', {}).get('outdated_offset', 5)
-        if latest_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))):
+        if latest_date < (arrow.utcnow().shift(minutes=-(timeframe_minutes * 2 + offset))):
             logger.warning(
                 'Outdated history for pair %s. Last tick is %s minutes old',
-                pair,
+                pair, int((arrow.utcnow() - latest_date).total_seconds() // 60)
-                (arrow.utcnow() - latest_date).seconds // 60
             )
             return False, False
 
         (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1
-        logger.debug(
+        logger.debug('trigger: %s (pair=%s) buy=%s sell=%s',
-            'trigger: %s (pair=%s) buy=%s sell=%s',
+                     latest['date'], pair, str(buy), str(sell))
-            latest['date'],
-            pair,
-            str(buy),
-            str(sell)
-        )
         return buy, sell
 
     def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool,
@@ -500,7 +599,8 @@ class IStrategy(ABC):
 
     def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
         """
-        Creates a dataframe and populates indicators for given candle (OHLCV) data
+        Populates indicators for given candle (OHLCV) data (for multiple pairs)
+        Does not run advice_buy or advise_sell!
         Used by optimize operations only, not during dry / live runs.
         Using .copy() to get a fresh copy of the dataframe for every strategy run.
         Has positive effects on memory usage for whatever reason - also when

freqtrade/strategy/strategy_wrapper.py

@@ -5,7 +5,7 @@ from freqtrade.exceptions import StrategyError
 logger = logging.getLogger(__name__)
 
 
-def strategy_safe_wrapper(f, message: str = "", default_retval=None):
+def strategy_safe_wrapper(f, message: str = "", default_retval=None, supress_error=False):
     """
     Wrapper around user-provided methods and functions.
     Caches all exceptions and returns either the default_retval (if it's not None) or raises
@@ -20,7 +20,7 @@ def strategy_safe_wrapper(f, message: str = "", default_retval=None):
                 f"Strategy caused the following exception: {error}"
                 f"{f}"
             )
-            if default_retval is None:
+            if default_retval is None and not supress_error:
                 raise StrategyError(str(error)) from error
             return default_retval
         except Exception as error:
@@ -28,7 +28,7 @@ def strategy_safe_wrapper(f, message: str = "", default_retval=None):
                 f"{message}"
                 f"Unexpected error {error} calling {f}"
             )
-            if default_retval is None:
+            if default_retval is None and not supress_error:
                 raise StrategyError(str(error)) from error
             return default_retval
 

freqtrade/templates/strategy_analysis_example.ipynb

@@ -34,7 +34,7 @@
     "# config = Configuration.from_files([\"config.json\"])\n",
     "\n",
     "# Define some constants\n",
-    "config[\"ticker_interval\"] = \"5m\"\n",
+    "config[\"timeframe\"] = \"5m\"\n",
     "# Name of the strategy class\n",
     "config[\"strategy\"] = \"SampleStrategy\"\n",
     "# Location of the data\n",
@@ -53,7 +53,7 @@
     "from freqtrade.data.history import load_pair_history\n",
     "\n",
     "candles = load_pair_history(datadir=data_location,\n",
-    "                            timeframe=config[\"ticker_interval\"],\n",
+    "                            timeframe=config[\"timeframe\"],\n",
     "                            pair=pair)\n",
     "\n",
     "# Confirm success\n",
@@ -136,10 +136,51 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from freqtrade.data.btanalysis import load_backtest_data\n",
+    "from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats\n",
     "\n",
-    "# Load backtest results\n",
+    "# if backtest_dir points to a directory, it'll automatically load the last backtest file.\n",
-    "trades = load_backtest_data(config[\"user_data_dir\"] / \"backtest_results/backtest-result.json\")\n",
+    "backtest_dir = config[\"user_data_dir\"] / \"backtest_results\"\n",
+    "# backtest_dir can also point to a specific file \n",
+    "# backtest_dir = config[\"user_data_dir\"] / \"backtest_results/backtest-result-2020-07-01_20-04-22.json\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# You can get the full backtest statistics by using the following command.\n",
+    "# This contains all information used to generate the backtest result.\n",
+    "stats = load_backtest_stats(backtest_dir)\n",
+    "\n",
+    "strategy = 'SampleStrategy'\n",
+    "# All statistics are available per strategy, so if `--strategy-list` was used during backtest, this will be reflected here as well.\n",
+    "# Example usages:\n",
+    "print(stats['strategy'][strategy]['results_per_pair'])\n",
+    "# Get pairlist used for this backtest\n",
+    "print(stats['strategy'][strategy]['pairlist'])\n",
+    "# Get market change (average change of all pairs from start to end of the backtest period)\n",
+    "print(stats['strategy'][strategy]['market_change'])\n",
+    "# Maximum drawdown ()\n",
+    "print(stats['strategy'][strategy]['max_drawdown'])\n",
+    "# Maximum drawdown start and end\n",
+    "print(stats['strategy'][strategy]['drawdown_start'])\n",
+    "print(stats['strategy'][strategy]['drawdown_end'])\n",
+    "\n",
+    "\n",
+    "# Get strategy comparison (only relevant if multiple strategies were compared)\n",
+    "print(stats['strategy_comparison'])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load backtested trades as dataframe\n",
+    "trades = load_backtest_data(backtest_dir)\n",
     "\n",
     "# Show value-counts per pair\n",
     "trades.groupby(\"pair\")[\"sell_reason\"].value_counts()"

freqtrade/templates/subtemplates/strategy_methods_advanced.j2

@@ -1,4 +1,65 @@
 
+def bot_loop_start(self, **kwargs) -> None:
+    """
+    Called at the start of the bot iteration (one loop).
+    Might be used to perform pair-independent tasks
+    (e.g. gather some remote ressource for comparison)
+
+    For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
+
+    When not implemented by a strategy, this simply does nothing.
+    :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+    """
+    pass
+
+def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float,
+                        time_in_force: str, **kwargs) -> bool:
+    """
+    Called right before placing a buy order.
+    Timing for this function is critical, so avoid doing heavy computations or
+    network requests in this method.
+
+    For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
+
+    When not implemented by a strategy, returns True (always confirming).
+
+    :param pair: Pair that's about to be bought.
+    :param order_type: Order type (as configured in order_types). usually limit or market.
+    :param amount: Amount in target (quote) currency that's going to be traded.
+    :param rate: Rate that's going to be used when using limit orders
+    :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
+    :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+    :return bool: When True is returned, then the buy-order is placed on the exchange.
+        False aborts the process
+    """
+    return True
+
+def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float,
+                       rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool:
+    """
+    Called right before placing a regular sell order.
+    Timing for this function is critical, so avoid doing heavy computations or
+    network requests in this method.
+
+    For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
+
+    When not implemented by a strategy, returns True (always confirming).
+
+    :param pair: Pair that's about to be sold.
+    :param trade: trade object.
+    :param order_type: Order type (as configured in order_types). usually limit or market.
+    :param amount: Amount in quote currency.
+    :param rate: Rate that's going to be used when using limit orders
+    :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled).
+    :param sell_reason: Sell reason.
+        Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss',
+                        'sell_signal', 'force_sell', 'emergency_sell']
+    :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
+    :return bool: When True is returned, then the sell-order is placed on the exchange.
+        False aborts the process
+    """
+    return True
+
 def check_buy_timeout(self, pair: str, trade: 'Trade', order: dict, **kwargs) -> bool:
     """
     Check buy timeout function callback.

freqtrade/vendor/qtpylib/indicators.py

@@ -222,7 +222,7 @@ def crossed(series1, series2, direction=None):
     if isinstance(series1, np.ndarray):
         series1 = pd.Series(series1)
 
-    if isinstance(series2, (float, int, np.ndarray)):
+    if isinstance(series2, (float, int, np.ndarray, np.integer, np.floating)):
         series2 = pd.Series(index=series1.index, data=series2)
 
     if direction is None or direction == "above":

freqtrade/worker.py

@@ -90,6 +90,9 @@ class Worker:
             if state == State.RUNNING:
                 self.freqtrade.startup()
 
+            if state == State.STOPPED:
+                self.freqtrade.check_for_open_trades()
+
             # Reset heartbeat timestamp to log the heartbeat message at
             # first throttling iteration when the state changes
             self._heartbeat_msg = 0

mkdocs.yml

@@ -3,6 +3,7 @@ nav:
     - Home: index.md
     - Installation Docker: docker.md
     - Installation: installation.md
+    - Freqtrade Basics: bot-basics.md
     - Configuration: configuration.md
     - Strategy Customization: strategy-customization.md
     - Stoploss: stoploss.md

requirements-common.txt

@@ -1,17 +1,17 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.30.48
+ccxt==1.33.52
-SQLAlchemy==1.3.18
+SQLAlchemy==1.3.19
 python-telegram-bot==12.8
-arrow==0.15.7
+arrow==0.16.0
 cachetools==4.1.1
 requests==2.24.0
-urllib3==1.25.9
+urllib3==1.25.10
 wrapt==1.12.1
 jsonschema==3.2.0
 TA-Lib==0.4.18
 tabulate==0.8.7
-pycoingecko==1.2.0
+pycoingecko==1.3.0
 jinja2==2.11.2
 
 # find first, C search in arrays
@@ -32,4 +32,4 @@ flask-cors==3.0.8
 colorama==0.4.3
 # Building config files interactively
 questionary==1.5.2
-prompt-toolkit==3.0.5
+prompt-toolkit==3.0.6

requirements-dev.txt

@@ -3,15 +3,15 @@
 -r requirements-plot.txt
 -r requirements-hyperopt.txt
 
-coveralls==2.0.0
+coveralls==2.1.2
 flake8==3.8.3
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==4.1.0
 mypy==0.782
-pytest==5.4.3
+pytest==6.0.1
 pytest-asyncio==0.14.0
-pytest-cov==2.10.0
+pytest-cov==2.10.1
-pytest-mock==3.1.1
+pytest-mock==3.3.0
 pytest-random-order==1.0.4
 
 # Convert jupyter notebooks to markdown documents

requirements-hyperopt.txt

@@ -2,9 +2,9 @@
 -r requirements.txt
 
 # Required for hyperopt
-scipy==1.5.0
+scipy==1.5.2
 scikit-learn==0.23.1
 scikit-optimize==0.7.4
 filelock==3.0.12
-joblib==0.15.1
+joblib==0.16.0
 progressbar2==3.51.4

requirements-plot.txt

@@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==4.8.2
+plotly==4.9.0
 

requirements.txt

@@ -1,5 +1,5 @@
 # Load common requirements
 -r requirements-common.txt
 
-numpy==1.19.0
+numpy==1.19.1
-pandas==1.0.5
+pandas==1.1.1

scripts/rest_client.py

@@ -62,6 +62,9 @@ class FtRestClient():
     def _get(self, apipath, params: dict = None):
         return self._call("GET", apipath, params=params)
 
+    def _delete(self, apipath, params: dict = None):
+        return self._call("DELETE", apipath, params=params)
+
     def _post(self, apipath, params: dict = None, data: dict = None):
         return self._call("POST", apipath, params=params, data=data)
 
@@ -156,6 +159,14 @@ class FtRestClient():
         """
         return self._get("show_config")
 
+    def logs(self, limit=None):
+        """Show latest logs.
+
+        :param limit: Limits log messages to the last <limit> logs. No limit to get all the trades.
+        :return: json object
+        """
+        return self._get("logs", params={"limit": limit} if limit else 0)
+
     def trades(self, limit=None):
         """Return trades history.
 
@@ -164,6 +175,15 @@ class FtRestClient():
         """
         return self._get("trades", params={"limit": limit} if limit else 0)
 
+    def delete_trade(self, trade_id):
+        """Delete trade from the database.
+        Tries to close open orders. Requires manual handling of this asset on the exchange.
+
+        :param trade_id: Deletes the trade with this ID from the database.
+        :return: json object
+        """
+        return self._delete("trades/{}".format(trade_id))
+
     def whitelist(self):
         """Show the current whitelist.
 
@@ -264,11 +284,11 @@ def main(args):
         print_commands()
         sys.exit()
 
-    config = load_config(args["config"])
+    config = load_config(args['config'])
-    url = config.get("api_server", {}).get("server_url", "127.0.0.1")
+    url = config.get('api_server', {}).get('server_url', '127.0.0.1')
-    port = config.get("api_server", {}).get("listen_port", "8080")
+    port = config.get('api_server', {}).get('listen_port', '8080')
-    username = config.get("api_server", {}).get("username")
+    username = config.get('api_server', {}).get('username')
-    password = config.get("api_server", {}).get("password")
+    password = config.get('api_server', {}).get('password')
 
     server_url = f"http://{url}:{port}"
     client = FtRestClient(server_url, username, password)

tests/commands/test_commands.py

@@ -6,12 +6,12 @@ import pytest
 
 from freqtrade.commands import (start_convert_data, start_create_userdir,
                                 start_download_data, start_hyperopt_list,
-                                start_hyperopt_show, start_list_exchanges,
+                                start_hyperopt_show, start_list_data,
-                                start_list_hyperopts, start_list_markets,
+                                start_list_exchanges, start_list_hyperopts,
-                                start_list_strategies, start_list_timeframes,
+                                start_list_markets, start_list_strategies,
-                                start_new_hyperopt, start_new_strategy,
+                                start_list_timeframes, start_new_hyperopt,
-                                start_show_trades, start_test_pairlist,
+                                start_new_strategy, start_show_trades,
-                                start_trading)
+                                start_test_pairlist, start_trading)
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.exceptions import OperationalException
 from freqtrade.state import RunMode
@@ -667,7 +667,7 @@ def test_start_list_hyperopts(mocker, caplog, capsys):
     args = [
         "list-hyperopts",
         "--hyperopt-path",
-        str(Path(__file__).parent.parent / "optimize"),
+        str(Path(__file__).parent.parent / "optimize" / "hyperopts"),
         "-1"
     ]
     pargs = get_args(args)
@@ -683,7 +683,7 @@ def test_start_list_hyperopts(mocker, caplog, capsys):
     args = [
         "list-hyperopts",
         "--hyperopt-path",
-        str(Path(__file__).parent.parent / "optimize"),
+        str(Path(__file__).parent.parent / "optimize" / "hyperopts"),
     ]
     pargs = get_args(args)
     # pargs['config'] = None
@@ -692,7 +692,6 @@ def test_start_list_hyperopts(mocker, caplog, capsys):
     assert "TestHyperoptLegacy" not in captured.out
     assert "legacy_hyperopt.py" not in captured.out
     assert "DefaultHyperOpt" in captured.out
-    assert "test_hyperopt.py" in captured.out
 
 
 def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys):
@@ -736,7 +735,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
 
     args = [
         "hyperopt-list",
-        "--no-details"
+        "--no-details",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -749,7 +748,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
     args = [
         "hyperopt-list",
         "--best",
-        "--no-details"
+        "--no-details",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -763,7 +762,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
     args = [
         "hyperopt-list",
         "--profitable",
-        "--no-details"
+        "--no-details",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -776,7 +775,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
                          " 11/12", " 12/12"])
     args = [
         "hyperopt-list",
-        "--profitable"
+        "--profitable",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -792,7 +791,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
         "hyperopt-list",
         "--no-details",
         "--no-color",
-        "--min-trades", "20"
+        "--min-trades", "20",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -806,7 +805,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
         "hyperopt-list",
         "--profitable",
         "--no-details",
-        "--max-trades", "20"
+        "--max-trades", "20",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -821,7 +820,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
         "hyperopt-list",
         "--profitable",
         "--no-details",
-        "--min-avg-profit", "0.11"
+        "--min-avg-profit", "0.11",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -835,7 +834,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
     args = [
         "hyperopt-list",
         "--no-details",
-        "--max-avg-profit", "0.10"
+        "--max-avg-profit", "0.10",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -849,7 +848,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
     args = [
         "hyperopt-list",
         "--no-details",
-        "--min-total-profit", "0.4"
+        "--min-total-profit", "0.4",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -863,7 +862,35 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
     args = [
         "hyperopt-list",
         "--no-details",
-        "--max-total-profit", "0.4"
+        "--max-total-profit", "0.4",
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_hyperopt_list(pargs)
+    captured = capsys.readouterr()
+    assert all(x in captured.out
+               for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12",
+                         " 9/12", " 11/12"])
+    assert all(x not in captured.out
+               for x in [" 4/12", " 10/12", " 12/12"])
+    args = [
+        "hyperopt-list",
+        "--no-details",
+        "--min-objective", "0.1",
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_hyperopt_list(pargs)
+    captured = capsys.readouterr()
+    assert all(x in captured.out
+               for x in [" 10/12"])
+    assert all(x not in captured.out
+               for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12",
+                         " 9/12", " 11/12", " 12/12"])
+    args = [
+        "hyperopt-list",
+        "--no-details",
+        "--max-objective", "0.1",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -878,7 +905,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
         "hyperopt-list",
         "--profitable",
         "--no-details",
-        "--min-avg-time", "2000"
+        "--min-avg-time", "2000",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -892,7 +919,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
     args = [
         "hyperopt-list",
         "--no-details",
-        "--max-avg-time", "1500"
+        "--max-avg-time", "1500",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -906,7 +933,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results):
     args = [
         "hyperopt-list",
         "--no-details",
-        "--export-csv", "test_file.csv"
+        "--export-csv", "test_file.csv",
     ]
     pargs = get_args(args)
     pargs['config'] = None
@@ -1043,6 +1070,40 @@ def test_convert_data_trades(mocker, testdatadir):
     assert trades_mock.call_args[1]['erase'] is False
 
 
+def test_start_list_data(testdatadir, capsys):
+    args = [
+        "list-data",
+        "--data-format-ohlcv",
+        "json",
+        "--datadir",
+        str(testdatadir),
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_list_data(pargs)
+    captured = capsys.readouterr()
+    assert "Found 16 pair / timeframe combinations." in captured.out
+    assert "\n|         Pair |       Timeframe |\n" in captured.out
+    assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m |\n" in captured.out
+
+    args = [
+        "list-data",
+        "--data-format-ohlcv",
+        "json",
+        "--pairs", "XRP/ETH",
+        "--datadir",
+        str(testdatadir),
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_list_data(pargs)
+    captured = capsys.readouterr()
+    assert "Found 2 pair / timeframe combinations." in captured.out
+    assert "\n|    Pair |   Timeframe |\n" in captured.out
+    assert "UNITTEST/BTC" not in captured.out
+    assert "\n| XRP/ETH |      1m, 5m |\n" in captured.out
+
+
 @pytest.mark.usefixtures("init_persistence")
 def test_show_trades(mocker, fee, capsys, caplog):
     mocker.patch("freqtrade.persistence.init")
@@ -1055,7 +1116,7 @@ def test_show_trades(mocker, fee, capsys, caplog):
     pargs = get_args(args)
     pargs['config'] = None
     start_show_trades(pargs)
-    assert log_has("Printing 3 Trades: ", caplog)
+    assert log_has("Printing 4 Trades: ", caplog)
     captured = capsys.readouterr()
     assert "Trade(id=1" in captured.out
     assert "Trade(id=2" in captured.out

tests/conftest.py

@@ -78,7 +78,7 @@ def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> No
 def get_patched_exchange(mocker, config, api_mock=None, id='bittrex',
                          mock_markets=True) -> Exchange:
     patch_exchange(mocker, api_mock, id, mock_markets)
-    config["exchange"]["name"] = id
+    config['exchange']['name'] = id
     try:
         exchange = ExchangeResolver.load_exchange(id, config)
     except ImportError:
@@ -163,7 +163,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     :param value: which value IStrategy.get_signal() must return
     :return: None
     """
-    freqtrade.strategy.get_signal = lambda e, s, t: value
+    freqtrade.strategy.get_signal = lambda e, s, x: value
     freqtrade.exchange.refresh_latest_ohlcv = lambda p: None
 
 
@@ -176,11 +176,13 @@ def create_mock_trades(fee):
         pair='ETH/BTC',
         stake_amount=0.001,
         amount=123.0,
+        amount_requested=123.0,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_rate=0.123,
         exchange='bittrex',
-        open_order_id='dry_run_buy_12345'
+        open_order_id='dry_run_buy_12345',
+        strategy='DefaultStrategy',
     )
     Trade.session.add(trade)
 
@@ -188,6 +190,7 @@ def create_mock_trades(fee):
         pair='ETC/BTC',
         stake_amount=0.001,
         amount=123.0,
+        amount_requested=123.0,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_rate=0.123,
@@ -195,7 +198,22 @@ def create_mock_trades(fee):
         close_profit=0.005,
         exchange='bittrex',
         is_open=False,
-        open_order_id='dry_run_sell_12345'
+        open_order_id='dry_run_sell_12345',
+        strategy='DefaultStrategy',
+    )
+    Trade.session.add(trade)
+
+    trade = Trade(
+        pair='XRP/BTC',
+        stake_amount=0.001,
+        amount=123.0,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.05,
+        close_rate=0.06,
+        close_profit=0.01,
+        exchange='bittrex',
+        is_open=False,
     )
     Trade.session.add(trade)
 
@@ -204,11 +222,13 @@ def create_mock_trades(fee):
         pair='ETC/BTC',
         stake_amount=0.001,
         amount=123.0,
+        amount_requested=124.0,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_rate=0.123,
         exchange='bittrex',
-        open_order_id='prod_buy_12345'
+        open_order_id='prod_buy_12345',
+        strategy='DefaultStrategy',
     )
     Trade.session.add(trade)
 
@@ -661,7 +681,8 @@ def shitcoinmarkets(markets):
     Fixture with shitcoin markets - used to test filters in pairlists
     """
     shitmarkets = deepcopy(markets)
-    shitmarkets.update({'HOT/BTC': {
+    shitmarkets.update({
+        'HOT/BTC': {
             'id': 'HOTBTC',
             'symbol': 'HOT/BTC',
             'base': 'HOT',
@@ -766,7 +787,32 @@ def shitcoinmarkets(markets):
             "spot": True,
             "future": False,
             "active": True
-    },
+        },
+        'ADADOUBLE/USDT': {
+            "percentage": True,
+            "tierBased": False,
+            "taker": 0.001,
+            "maker": 0.001,
+            "precision": {
+                "base": 8,
+                "quote": 8,
+                "amount": 2,
+                "price": 4
+            },
+            "limits": {
+            },
+            "id": "ADADOUBLEUSDT",
+            "symbol": "ADADOUBLE/USDT",
+            "base": "ADADOUBLE",
+            "quote": "USDT",
+            "baseId": "ADADOUBLE",
+            "quoteId": "USDT",
+            "info": {},
+            "type": "spot",
+            "spot": True,
+            "future": False,
+            "active": True
+        },
         })
     return shitmarkets
 
@@ -787,6 +833,7 @@ def limit_buy_order():
         'price': 0.00001099,
         'amount': 90.99181073,
         'filled': 90.99181073,
+        'cost': 0.0009999,
         'remaining': 0.0,
         'status': 'closed'
     }
@@ -1387,6 +1434,28 @@ def tickers():
             "quoteVolume": 0.0,
             "info": {}
         },
+        "ADADOUBLE/USDT": {
+            "symbol": "ADADOUBLE/USDT",
+            "timestamp": 1580469388244,
+            "datetime": "2020-01-31T11:16:28.244Z",
+            "high": None,
+            "low": None,
+            "bid": 0.7305,
+            "bidVolume": None,
+            "ask": 0.7342,
+            "askVolume": None,
+            "vwap": None,
+            "open": None,
+            "close": None,
+            "last": 0,
+            "previousClose": None,
+            "change": None,
+            "percentage": 2.628,
+            "average": None,
+            "baseVolume": 0.0,
+            "quoteVolume": 0.0,
+            "info": {}
+        },
     })
 
 

tests/data/test_btanalysis.py

@@ -6,24 +6,48 @@ from arrow import Arrow
 from pandas import DataFrame, DateOffset, Timestamp, to_datetime
 
 from freqtrade.configuration import TimeRange
+from freqtrade.constants import LAST_BT_RESULT_FN
 from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
                                        analyze_trade_parallelism,
+                                       calculate_market_change,
                                        calculate_max_drawdown,
                                        combine_dataframes_with_mean,
                                        create_cum_profit,
                                        extract_trades_of_period,
+                                       get_latest_backtest_filename,
                                        load_backtest_data, load_trades,
                                        load_trades_from_db)
 from freqtrade.data.history import load_data, load_pair_history
+from freqtrade.optimize.backtesting import BacktestResult
 from tests.conftest import create_mock_trades
 
 
-def test_load_backtest_data(testdatadir):
+def test_get_latest_backtest_filename(testdatadir, mocker):
+    with pytest.raises(ValueError, match=r"Directory .* does not exist\."):
+        get_latest_backtest_filename(testdatadir / 'does_not_exist')
+
+    with pytest.raises(ValueError,
+                       match=r"Directory .* does not seem to contain .*"):
+        get_latest_backtest_filename(testdatadir.parent)
+
+    res = get_latest_backtest_filename(testdatadir)
+    assert res == 'backtest-result_new.json'
+
+    res = get_latest_backtest_filename(str(testdatadir))
+    assert res == 'backtest-result_new.json'
+
+    mocker.patch("freqtrade.data.btanalysis.json_load", return_value={})
+
+    with pytest.raises(ValueError, match=r"Invalid '.last_result.json' format."):
+        get_latest_backtest_filename(testdatadir)
+
+
+def test_load_backtest_data_old_format(testdatadir):
 
     filename = testdatadir / "backtest-result_test.json"
     bt_data = load_backtest_data(filename)
     assert isinstance(bt_data, DataFrame)
-    assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit"]
+    assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit_abs"]
     assert len(bt_data) == 179
 
     # Test loading from string (must yield same result)
@@ -34,6 +58,49 @@ def test_load_backtest_data(testdatadir):
         load_backtest_data(str("filename") + "nofile")
 
 
+def test_load_backtest_data_new_format(testdatadir):
+
+    filename = testdatadir / "backtest-result_new.json"
+    bt_data = load_backtest_data(filename)
+    assert isinstance(bt_data, DataFrame)
+    assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"])
+    assert len(bt_data) == 179
+
+    # Test loading from string (must yield same result)
+    bt_data2 = load_backtest_data(str(filename))
+    assert bt_data.equals(bt_data2)
+
+    # Test loading from folder (must yield same result)
+    bt_data3 = load_backtest_data(testdatadir)
+    assert bt_data.equals(bt_data3)
+
+    with pytest.raises(ValueError, match=r"File .* does not exist\."):
+        load_backtest_data(str("filename") + "nofile")
+
+    with pytest.raises(ValueError, match=r"Unknown dataformat."):
+        load_backtest_data(testdatadir / LAST_BT_RESULT_FN)
+
+
+def test_load_backtest_data_multi(testdatadir):
+
+    filename = testdatadir / "backtest-result_multistrat.json"
+    for strategy in ('DefaultStrategy', 'TestStrategy'):
+        bt_data = load_backtest_data(filename, strategy=strategy)
+        assert isinstance(bt_data, DataFrame)
+        assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"])
+        assert len(bt_data) == 179
+
+        # Test loading from string (must yield same result)
+        bt_data2 = load_backtest_data(str(filename), strategy=strategy)
+        assert bt_data.equals(bt_data2)
+
+    with pytest.raises(ValueError, match=r"Strategy XYZ not available in the backtest result\."):
+        load_backtest_data(filename, strategy='XYZ')
+
+    with pytest.raises(ValueError, match=r"Detected backtest result with more than one strategy.*"):
+        load_backtest_data(filename)
+
+
 @pytest.mark.usefixtures("init_persistence")
 def test_load_trades_from_db(default_conf, fee, mocker):
 
@@ -43,15 +110,19 @@ def test_load_trades_from_db(default_conf, fee, mocker):
 
     trades = load_trades_from_db(db_url=default_conf['db_url'])
     assert init_mock.call_count == 1
-    assert len(trades) == 3
+    assert len(trades) == 4
     assert isinstance(trades, DataFrame)
     assert "pair" in trades.columns
-    assert "open_time" in trades.columns
+    assert "open_date" in trades.columns
     assert "profit_percent" in trades.columns
 
     for col in BT_DATA_COLUMNS:
         if col not in ['index', 'open_at_end']:
             assert col in trades.columns
+    trades = load_trades_from_db(db_url=default_conf['db_url'], strategy='DefaultStrategy')
+    assert len(trades) == 3
+    trades = load_trades_from_db(db_url=default_conf['db_url'], strategy='NoneStrategy')
+    assert len(trades) == 0
 
 
 def test_extract_trades_of_period(testdatadir):
@@ -66,13 +137,13 @@ def test_extract_trades_of_period(testdatadir):
         {'pair': [pair, pair, pair, pair],
          'profit_percent': [0.0, 0.1, -0.2, -0.5],
          'profit_abs': [0.0, 1, -2, -5],
-         'open_time': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime,
+         'open_date': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime,
                                    Arrow(2017, 11, 14, 9, 41, 0).datetime,
                                    Arrow(2017, 11, 14, 14, 20, 0).datetime,
                                    Arrow(2017, 11, 15, 3, 40, 0).datetime,
                                    ], utc=True
                                   ),
-         'close_time': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime,
+         'close_date': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime,
                                     Arrow(2017, 11, 14, 10, 41, 0).datetime,
                                     Arrow(2017, 11, 14, 15, 25, 0).datetime,
                                     Arrow(2017, 11, 15, 3, 55, 0).datetime,
@@ -81,10 +152,10 @@ def test_extract_trades_of_period(testdatadir):
     trades1 = extract_trades_of_period(data, trades)
     # First and last trade are dropped as they are out of range
     assert len(trades1) == 2
-    assert trades1.iloc[0].open_time == Arrow(2017, 11, 14, 9, 41, 0).datetime
+    assert trades1.iloc[0].open_date == Arrow(2017, 11, 14, 9, 41, 0).datetime
-    assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime
+    assert trades1.iloc[0].close_date == Arrow(2017, 11, 14, 10, 41, 0).datetime
-    assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime
+    assert trades1.iloc[-1].open_date == Arrow(2017, 11, 14, 14, 20, 0).datetime
-    assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime
+    assert trades1.iloc[-1].close_date == Arrow(2017, 11, 14, 15, 25, 0).datetime
 
 
 def test_analyze_trade_parallelism(default_conf, mocker, testdatadir):
@@ -105,7 +176,8 @@ def test_load_trades(default_conf, mocker):
     load_trades("DB",
                 db_url=default_conf.get('db_url'),
                 exportfilename=default_conf.get('exportfilename'),
-                no_trades=False
+                no_trades=False,
+                strategy="DefaultStrategy",
                 )
 
     assert db_mock.call_count == 1
@@ -135,6 +207,14 @@ def test_load_trades(default_conf, mocker):
     assert bt_mock.call_count == 0
 
 
+def test_calculate_market_change(testdatadir):
+    pairs = ["ETH/BTC", "ADA/BTC"]
+    data = load_data(datadir=testdatadir, pairs=pairs, timeframe='5m')
+    result = calculate_market_change(data)
+    assert isinstance(result, float)
+    assert pytest.approx(result) == 0.00955514
+
+
 def test_combine_dataframes_with_mean(testdatadir):
     pairs = ["ETH/BTC", "ADA/BTC"]
     data = load_data(datadir=testdatadir, pairs=pairs, timeframe='5m')
@@ -165,7 +245,7 @@ def test_create_cum_profit1(testdatadir):
     filename = testdatadir / "backtest-result_test.json"
     bt_data = load_backtest_data(filename)
     # Move close-time to "off" the candle, to make sure the logic still works
-    bt_data.loc[:, 'close_time'] = bt_data.loc[:, 'close_time'] + DateOffset(seconds=20)
+    bt_data.loc[:, 'close_date'] = bt_data.loc[:, 'close_date'] + DateOffset(seconds=20)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
     df = load_pair_history(pair="TRX/BTC", timeframe='5m',
@@ -204,11 +284,11 @@ def test_calculate_max_drawdown2():
               -0.033961, 0.010680, 0.010886, -0.029274, 0.011178, 0.010693, 0.010711]
 
     dates = [Arrow(2020, 1, 1).shift(days=i) for i in range(len(values))]
-    df = DataFrame(zip(values, dates), columns=['profit', 'open_time'])
+    df = DataFrame(zip(values, dates), columns=['profit', 'open_date'])
     # sort by profit and reset index
     df = df.sort_values('profit').reset_index(drop=True)
     df1 = df.copy()
-    drawdown, h, low = calculate_max_drawdown(df, date_col='open_time', value_col='profit')
+    drawdown, h, low = calculate_max_drawdown(df, date_col='open_date', value_col='profit')
     # Ensure df has not been altered.
     assert df.equals(df1)
 
@@ -217,6 +297,6 @@ def test_calculate_max_drawdown2():
     assert h < low
     assert drawdown == 0.091755
 
-    df = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_time'])
+    df = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_date'])
     with pytest.raises(ValueError, match='No losing trade, therefore no drawdown.'):
-        calculate_max_drawdown(df, date_col='open_time', value_col='profit')
+        calculate_max_drawdown(df, date_col='open_date', value_col='profit')

tests/data/test_dataprovider.py

@@ -1,11 +1,12 @@
+from datetime import datetime, timezone
 from unittest.mock import MagicMock
 
-from pandas import DataFrame
 import pytest
+from pandas import DataFrame
 
 from freqtrade.data.dataprovider import DataProvider
+from freqtrade.exceptions import ExchangeError, OperationalException
 from freqtrade.pairlist.pairlistmanager import PairListManager
-from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.state import RunMode
 from tests.conftest import get_patched_exchange
 
@@ -164,7 +165,7 @@ def test_ticker(mocker, default_conf, tickers):
     assert 'symbol' in res
     assert res['symbol'] == 'ETH/BTC'
 
-    ticker_mock = MagicMock(side_effect=DependencyException('Pair not found'))
+    ticker_mock = MagicMock(side_effect=ExchangeError('Pair not found'))
     mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock)
     exchange = get_patched_exchange(mocker, default_conf)
     dp = DataProvider(default_conf, exchange)
@@ -194,3 +195,29 @@ def test_current_whitelist(mocker, default_conf, tickers):
     with pytest.raises(OperationalException):
         dp = DataProvider(default_conf, exchange)
         dp.current_whitelist()
+
+
+def test_get_analyzed_dataframe(mocker, default_conf, ohlcv_history):
+
+    default_conf["runmode"] = RunMode.DRY_RUN
+
+    timeframe = default_conf["timeframe"]
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    dp = DataProvider(default_conf, exchange)
+    dp._set_cached_df("XRP/BTC", timeframe, ohlcv_history)
+    dp._set_cached_df("UNITTEST/BTC", timeframe, ohlcv_history)
+
+    assert dp.runmode == RunMode.DRY_RUN
+    dataframe, time = dp.get_analyzed_dataframe("UNITTEST/BTC", timeframe)
+    assert ohlcv_history.equals(dataframe)
+    assert isinstance(time, datetime)
+
+    dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe)
+    assert ohlcv_history.equals(dataframe)
+    assert isinstance(time, datetime)
+
+    dataframe, time = dp.get_analyzed_dataframe("NOTHING/BTC", timeframe)
+    assert dataframe.empty
+    assert isinstance(time, datetime)
+    assert time == datetime(1970, 1, 1, tzinfo=timezone.utc)

tests/data/test_history.py

@@ -36,7 +36,7 @@ def _backup_file(file: Path, copy_file: bool = False) -> None:
     """
     Backup existing file to avoid deleting the user file
     :param file: complete path to the file
-    :param touch_file: create an empty file in replacement
+    :param copy_file: keep file in place too.
     :return: None
     """
     file_swp = str(file) + '.swp'
@@ -631,6 +631,20 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir):
     assert set(pairs) == {'UNITTEST/BTC'}
 
 
+def test_jsondatahandler_ohlcv_get_available_data(testdatadir):
+    paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir)
+    # Convert to set to avoid failures due to sorting
+    assert set(paircombs) == {('UNITTEST/BTC', '5m'), ('ETH/BTC', '5m'), ('XLM/BTC', '5m'),
+                              ('TRX/BTC', '5m'), ('LTC/BTC', '5m'), ('XMR/BTC', '5m'),
+                              ('ZEC/BTC', '5m'), ('UNITTEST/BTC', '1m'), ('ADA/BTC', '5m'),
+                              ('ETC/BTC', '5m'), ('NXT/BTC', '5m'), ('DASH/BTC', '5m'),
+                              ('XRP/ETH', '1m'), ('XRP/ETH', '5m'), ('UNITTEST/BTC', '30m'),
+                              ('UNITTEST/BTC', '8m')}
+
+    paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir)
+    assert set(paircombs) == {('UNITTEST/BTC', '8m')}
+
+
 def test_jsondatahandler_trades_get_pairs(testdatadir):
     pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
     # Convert to set to avoid failures due to sorting

tests/edge/test_edge.py

@@ -163,8 +163,8 @@ def test_edge_results(edge_conf, mocker, caplog, data) -> None:
     for c, trade in enumerate(data.trades):
         res = results.iloc[c]
         assert res.exit_type == trade.sell_reason
-        assert res.open_time == _get_frame_time_from_offset(trade.open_tick).replace(tzinfo=None)
+        assert res.open_date == _get_frame_time_from_offset(trade.open_tick).replace(tzinfo=None)
-        assert res.close_time == _get_frame_time_from_offset(trade.close_tick).replace(tzinfo=None)
+        assert res.close_date == _get_frame_time_from_offset(trade.close_tick).replace(tzinfo=None)
 
 
 def test_adjust(mocker, edge_conf):
@@ -354,10 +354,8 @@ def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectanc
          'stoploss': -0.9,
          'profit_percent': '',
          'profit_abs': '',
-         'open_time': np.datetime64('2018-10-03T00:05:00.000000000'),
+         'open_date': np.datetime64('2018-10-03T00:05:00.000000000'),
-         'close_time': np.datetime64('2018-10-03T00:10:00.000000000'),
+         'close_date': np.datetime64('2018-10-03T00:10:00.000000000'),
-         'open_index': 1,
-         'close_index': 1,
          'trade_duration': '',
          'open_rate': 17,
          'close_rate': 17,
@@ -367,10 +365,8 @@ def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectanc
          'stoploss': -0.9,
          'profit_percent': '',
          'profit_abs': '',
-         'open_time': np.datetime64('2018-10-03T00:20:00.000000000'),
+         'open_date': np.datetime64('2018-10-03T00:20:00.000000000'),
-         'close_time': np.datetime64('2018-10-03T00:25:00.000000000'),
+         'close_date': np.datetime64('2018-10-03T00:25:00.000000000'),
-         'open_index': 4,
-         'close_index': 4,
          'trade_duration': '',
          'open_rate': 20,
          'close_rate': 20,
@@ -380,10 +376,8 @@ def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectanc
          'stoploss': -0.9,
          'profit_percent': '',
          'profit_abs': '',
-         'open_time': np.datetime64('2018-10-03T00:30:00.000000000'),
+         'open_date': np.datetime64('2018-10-03T00:30:00.000000000'),
-         'close_time': np.datetime64('2018-10-03T00:40:00.000000000'),
+         'close_date': np.datetime64('2018-10-03T00:40:00.000000000'),
-         'open_index': 6,
-         'close_index': 7,
          'trade_duration': '',
          'open_rate': 26,
          'close_rate': 34,
@@ -409,3 +403,98 @@ def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectanc
     final = edge._process_expectancy(trades_df)
     assert len(final) == 0
     assert isinstance(final, dict)
+
+
+def test_process_expectancy_remove_pumps(mocker, edge_conf, fee,):
+    edge_conf['edge']['min_trade_number'] = 2
+    edge_conf['edge']['remove_pumps'] = True
+    freqtrade = get_patched_freqtradebot(mocker, edge_conf)
+
+    freqtrade.exchange.get_fee = fee
+    edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
+
+    trades = [
+        {'pair': 'TEST/BTC',
+         'stoploss': -0.9,
+         'profit_percent': '',
+         'profit_abs': '',
+         'open_date': np.datetime64('2018-10-03T00:05:00.000000000'),
+         'close_date': np.datetime64('2018-10-03T00:10:00.000000000'),
+         'open_index': 1,
+         'close_index': 1,
+         'trade_duration': '',
+         'open_rate': 17,
+         'close_rate': 15,
+         'exit_type': 'sell_signal'},
+
+        {'pair': 'TEST/BTC',
+         'stoploss': -0.9,
+         'profit_percent': '',
+         'profit_abs': '',
+         'open_date': np.datetime64('2018-10-03T00:20:00.000000000'),
+         'close_date': np.datetime64('2018-10-03T00:25:00.000000000'),
+         'open_index': 4,
+         'close_index': 4,
+         'trade_duration': '',
+         'open_rate': 20,
+         'close_rate': 10,
+         'exit_type': 'sell_signal'},
+        {'pair': 'TEST/BTC',
+         'stoploss': -0.9,
+         'profit_percent': '',
+         'profit_abs': '',
+         'open_date': np.datetime64('2018-10-03T00:20:00.000000000'),
+         'close_date': np.datetime64('2018-10-03T00:25:00.000000000'),
+         'open_index': 4,
+         'close_index': 4,
+         'trade_duration': '',
+         'open_rate': 20,
+         'close_rate': 10,
+         'exit_type': 'sell_signal'},
+        {'pair': 'TEST/BTC',
+         'stoploss': -0.9,
+         'profit_percent': '',
+         'profit_abs': '',
+         'open_date': np.datetime64('2018-10-03T00:20:00.000000000'),
+         'close_date': np.datetime64('2018-10-03T00:25:00.000000000'),
+         'open_index': 4,
+         'close_index': 4,
+         'trade_duration': '',
+         'open_rate': 20,
+         'close_rate': 10,
+         'exit_type': 'sell_signal'},
+        {'pair': 'TEST/BTC',
+         'stoploss': -0.9,
+         'profit_percent': '',
+         'profit_abs': '',
+         'open_date': np.datetime64('2018-10-03T00:20:00.000000000'),
+         'close_date': np.datetime64('2018-10-03T00:25:00.000000000'),
+         'open_index': 4,
+         'close_index': 4,
+         'trade_duration': '',
+         'open_rate': 20,
+         'close_rate': 10,
+         'exit_type': 'sell_signal'},
+
+        {'pair': 'TEST/BTC',
+         'stoploss': -0.9,
+         'profit_percent': '',
+         'profit_abs': '',
+         'open_date': np.datetime64('2018-10-03T00:30:00.000000000'),
+         'close_date': np.datetime64('2018-10-03T00:40:00.000000000'),
+         'open_index': 6,
+         'close_index': 7,
+         'trade_duration': '',
+         'open_rate': 26,
+         'close_rate': 134,
+         'exit_type': 'sell_signal'}
+    ]
+
+    trades_df = DataFrame(trades)
+    trades_df = edge._fill_calculable_fields(trades_df)
+    final = edge._process_expectancy(trades_df)
+
+    assert 'TEST/BTC' in final
+    assert final['TEST/BTC'].stoploss == -0.9
+    assert final['TEST/BTC'].nb_trades == len(trades_df) - 1
+    assert round(final['TEST/BTC'].winrate, 10) == 0.0

tests/exchange/test_binance.py

@@ -5,8 +5,9 @@ import ccxt
 import pytest
 
 from freqtrade.exceptions import (DependencyException, InvalidOrderException,
-                                  OperationalException, TemporaryError)
+                                  OperationalException)
 from tests.conftest import get_patched_exchange
+from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
 
 @pytest.mark.parametrize('limitratio,expected', [
@@ -62,15 +63,9 @@ def test_stoploss_order_binance(default_conf, mocker, limitratio, expected):
         exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
         exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
 
-    with pytest.raises(TemporaryError):
+    ccxt_exceptionhandlers(mocker, default_conf, api_mock, "binance",
-        api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection"))
+                           "stoploss", "create_order", retries=1,
-        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
+                           pair='ETH/BTC', amount=1, stop_price=220, order_types={})
-        exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
-
-    with pytest.raises(OperationalException, match=r".*DeadBeef.*"):
-        api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef"))
-        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance')
-        exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
 
 
 def test_stoploss_order_dry_run_binance(default_conf, mocker):

tests/exchange/test_exchange.py

@@ -4,18 +4,19 @@ import copy
 import logging
 from datetime import datetime, timezone
 from random import randint
-from unittest.mock import MagicMock, Mock, PropertyMock
+from unittest.mock import MagicMock, Mock, PropertyMock, patch
 
 import arrow
 import ccxt
 import pytest
 from pandas import DataFrame
 
-from freqtrade.exceptions import (DependencyException, InvalidOrderException,
+from freqtrade.exceptions import (DDosProtection, DependencyException,
-                                  OperationalException, TemporaryError)
+                                  InvalidOrderException, OperationalException,
+                                  TemporaryError)
 from freqtrade.exchange import Binance, Exchange, Kraken
-from freqtrade.exchange.common import API_RETRY_COUNT
+from freqtrade.exchange.common import API_RETRY_COUNT, calculate_backoff
-from freqtrade.exchange.exchange import (market_is_active, symbol_is_pair,
+from freqtrade.exchange.exchange import (market_is_active,
                                          timeframe_to_minutes,
                                          timeframe_to_msecs,
                                          timeframe_to_next_date,
@@ -37,12 +38,20 @@ def get_mock_coro(return_value):
 
 
 def ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
-                           fun, mock_ccxt_fun, **kwargs):
+                           fun, mock_ccxt_fun, retries=API_RETRY_COUNT + 1, **kwargs):
+
+    with patch('freqtrade.exchange.common.time.sleep'):
+        with pytest.raises(DDosProtection):
+            api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.DDoSProtection("DDos"))
+            exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+            getattr(exchange, fun)(**kwargs)
+        assert api_mock.__dict__[mock_ccxt_fun].call_count == retries
+
     with pytest.raises(TemporaryError):
         api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError("DeaDBeef"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
         getattr(exchange, fun)(**kwargs)
-    assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1
+    assert api_mock.__dict__[mock_ccxt_fun].call_count == retries
 
     with pytest.raises(OperationalException):
         api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError("DeadBeef"))
@@ -51,12 +60,21 @@ def ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
     assert api_mock.__dict__[mock_ccxt_fun].call_count == 1
 
 
-async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs):
+async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun,
+                               retries=API_RETRY_COUNT + 1, **kwargs):
+
+    with patch('freqtrade.exchange.common.asyncio.sleep', get_mock_coro(None)):
+        with pytest.raises(DDosProtection):
+            api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.DDoSProtection("Dooh"))
+            exchange = get_patched_exchange(mocker, default_conf, api_mock)
+            await getattr(exchange, fun)(**kwargs)
+        assert api_mock.__dict__[mock_ccxt_fun].call_count == retries
+
     with pytest.raises(TemporaryError):
         api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError("DeadBeef"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock)
         await getattr(exchange, fun)(**kwargs)
-    assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1
+    assert api_mock.__dict__[mock_ccxt_fun].call_count == retries
 
     with pytest.raises(OperationalException):
         api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError("DeadBeef"))
@@ -697,13 +715,13 @@ def test_validate_order_types(default_conf, mocker):
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
     mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
     mocker.patch('freqtrade.exchange.Exchange.name', 'Bittrex')
+
     default_conf['order_types'] = {
         'buy': 'limit',
         'sell': 'limit',
         'stoploss': 'market',
         'stoploss_on_exchange': False
     }
-
     Exchange(default_conf)
 
     type(api_mock).has = PropertyMock(return_value={'createMarketOrder': False})
@@ -713,9 +731,8 @@ def test_validate_order_types(default_conf, mocker):
         'buy': 'limit',
         'sell': 'limit',
         'stoploss': 'market',
-        'stoploss_on_exchange': 'false'
+        'stoploss_on_exchange': False
     }
-
     with pytest.raises(OperationalException,
                        match=r'Exchange .* does not support market orders.'):
         Exchange(default_conf)
@@ -726,7 +743,6 @@ def test_validate_order_types(default_conf, mocker):
         'stoploss': 'limit',
         'stoploss_on_exchange': True
     }
-
     with pytest.raises(OperationalException,
                        match=r'On exchange stoploss is not supported for .*'):
         Exchange(default_conf)
@@ -1127,9 +1143,10 @@ def test_get_balance_prod(default_conf, mocker, exchange_name):
         exchange.get_balance(currency='BTC')
 
 
-def test_get_balances_dry_run(default_conf, mocker):
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_get_balances_dry_run(default_conf, mocker, exchange_name):
     default_conf['dry_run'] = True
-    exchange = get_patched_exchange(mocker, default_conf)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
     assert exchange.get_balances() == {}
 
 
@@ -1802,7 +1819,7 @@ def test_cancel_order_with_result_error(default_conf, mocker, exchange_name, cap
 
     res = exchange.cancel_order_with_result('1234', 'ETH/BTC', 1541)
     assert isinstance(res, dict)
-    assert log_has("Could not cancel order 1234.", caplog)
+    assert log_has("Could not cancel order 1234 for ETH/BTC.", caplog)
     assert log_has("Could not fetch cancelled order 1234.", caplog)
     assert res['amount'] == 1541
 
@@ -1847,36 +1864,48 @@ def test_cancel_stoploss_order(default_conf, mocker, exchange_name):
 
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
-def test_get_order(default_conf, mocker, exchange_name):
+def test_fetch_order(default_conf, mocker, exchange_name):
     default_conf['dry_run'] = True
     order = MagicMock()
     order.myid = 123
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
     exchange._dry_run_open_orders['X'] = order
-    assert exchange.get_order('X', 'TKN/BTC').myid == 123
+    assert exchange.fetch_order('X', 'TKN/BTC').myid == 123
 
     with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'):
-        exchange.get_order('Y', 'TKN/BTC')
+        exchange.fetch_order('Y', 'TKN/BTC')
 
     default_conf['dry_run'] = False
     api_mock = MagicMock()
     api_mock.fetch_order = MagicMock(return_value=456)
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-    assert exchange.get_order('X', 'TKN/BTC') == 456
+    assert exchange.fetch_order('X', 'TKN/BTC') == 456
 
     with pytest.raises(InvalidOrderException):
         api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.get_order(order_id='_', pair='TKN/BTC')
+        exchange.fetch_order(order_id='_', pair='TKN/BTC')
     assert api_mock.fetch_order.call_count == 1
 
+    api_mock.fetch_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found"))
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+    with patch('freqtrade.exchange.common.time.sleep') as tm:
+        with pytest.raises(InvalidOrderException):
+            exchange.fetch_order(order_id='_', pair='TKN/BTC')
+        # Ensure backoff is called
+        assert tm.call_args_list[0][0][0] == 1
+        assert tm.call_args_list[1][0][0] == 2
+        assert tm.call_args_list[2][0][0] == 5
+        assert tm.call_args_list[3][0][0] == 10
+    assert api_mock.fetch_order.call_count == 6
+
     ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
-                           'get_order', 'fetch_order',
+                           'fetch_order', 'fetch_order', retries=6,
                            order_id='_', pair='TKN/BTC')
 
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
-def test_get_stoploss_order(default_conf, mocker, exchange_name):
+def test_fetch_stoploss_order(default_conf, mocker, exchange_name):
     # Don't test FTX here - that needs a seperate test
     if exchange_name == 'ftx':
         return
@@ -1885,25 +1914,26 @@ def test_get_stoploss_order(default_conf, mocker, exchange_name):
     order.myid = 123
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
     exchange._dry_run_open_orders['X'] = order
-    assert exchange.get_stoploss_order('X', 'TKN/BTC').myid == 123
+    assert exchange.fetch_stoploss_order('X', 'TKN/BTC').myid == 123
 
     with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'):
-        exchange.get_stoploss_order('Y', 'TKN/BTC')
+        exchange.fetch_stoploss_order('Y', 'TKN/BTC')
 
     default_conf['dry_run'] = False
     api_mock = MagicMock()
     api_mock.fetch_order = MagicMock(return_value=456)
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-    assert exchange.get_stoploss_order('X', 'TKN/BTC') == 456
+    assert exchange.fetch_stoploss_order('X', 'TKN/BTC') == 456
 
     with pytest.raises(InvalidOrderException):
         api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
-        exchange.get_stoploss_order(order_id='_', pair='TKN/BTC')
+        exchange.fetch_stoploss_order(order_id='_', pair='TKN/BTC')
     assert api_mock.fetch_order.call_count == 1
 
     ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
-                           'get_stoploss_order', 'fetch_order',
+                           'fetch_stoploss_order', 'fetch_order',
+                           retries=6,
                            order_id='_', pair='TKN/BTC')
 
 
@@ -2111,6 +2141,13 @@ def test_get_markets(default_conf, mocker, markets,
     assert sorted(pairs.keys()) == sorted(expected_keys)
 
 
+def test_get_markets_error(default_conf, mocker):
+    ex = get_patched_exchange(mocker, default_conf)
+    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=None))
+    with pytest.raises(OperationalException, match="Markets were not loaded."):
+        ex.get_markets('LTC', 'USDT', True, False)
+
+
 def test_timeframe_to_minutes():
     assert timeframe_to_minutes("5m") == 5
     assert timeframe_to_minutes("10m") == 10
@@ -2182,25 +2219,42 @@ def test_timeframe_to_next_date():
     assert timeframe_to_next_date("5m") > date
 
 
-@pytest.mark.parametrize("market_symbol,base_currency,quote_currency,expected_result", [
+@pytest.mark.parametrize("market_symbol,base,quote,exchange,add_dict,expected_result", [
-    ("BTC/USDT", None, None, True),
+    ("BTC/USDT", 'BTC', 'USDT', "binance", {}, True),
-    ("USDT/BTC", None, None, True),
+    ("USDT/BTC", 'USDT', 'BTC', "binance", {}, True),
-    ("BTCUSDT", None, None, False),
+    ("USDT/BTC", 'BTC', 'USDT', "binance", {}, False),  # Reversed currencies
-    ("BTC/USDT", None, "USDT", True),
+    ("BTCUSDT", 'BTC', 'USDT', "binance", {}, False),  # No seperating /
-    ("USDT/BTC", None, "USDT", False),
+    ("BTCUSDT", None, "USDT", "binance", {}, False),  #
-    ("BTCUSDT", None, "USDT", False),
+    ("USDT/BTC", "BTC", None, "binance", {}, False),
-    ("BTC/USDT", "BTC", None, True),
+    ("BTCUSDT", "BTC", None, "binance", {}, False),
-    ("USDT/BTC", "BTC", None, False),
+    ("BTC/USDT", "BTC", "USDT", "binance", {}, True),
-    ("BTCUSDT", "BTC", None, False),
+    ("BTC/USDT", "USDT", "BTC", "binance", {}, False),  # reversed currencies
-    ("BTC/USDT", "BTC", "USDT", True),
+    ("BTC/USDT", "BTC", "USD", "binance", {}, False),  # Wrong quote currency
-    ("BTC/USDT", "USDT", "BTC", False),
+    ("BTC/", "BTC", 'UNK', "binance", {}, False),
-    ("BTC/USDT", "BTC", "USD", False),
+    ("/USDT", 'UNK', 'USDT', "binance", {}, False),
-    ("BTCUSDT", "BTC", "USDT", False),
+    ("BTC/EUR", 'BTC', 'EUR', "kraken", {"darkpool": False}, True),
-    ("BTC/", None, None, False),
+    ("EUR/BTC", 'EUR', 'BTC', "kraken", {"darkpool": False}, True),
-    ("/USDT", None, None, False),
+    ("EUR/BTC", 'BTC', 'EUR', "kraken", {"darkpool": False}, False),  # Reversed currencies
+    ("BTC/EUR", 'BTC', 'USD', "kraken", {"darkpool": False}, False),  # wrong quote currency
+    ("BTC/EUR", 'BTC', 'EUR', "kraken", {"darkpool": True}, False),  # no darkpools
+    ("BTC/EUR.d", 'BTC', 'EUR', "kraken", {"darkpool": True}, False),  # no darkpools
+    ("BTC/USD", 'BTC', 'USD', "ftx", {'spot': True}, True),
+    ("USD/BTC", 'USD', 'BTC', "ftx", {'spot': True}, True),
+    ("BTC/USD", 'BTC', 'USDT', "ftx", {'spot': True}, False),  # Wrong quote currency
+    ("BTC/USD", 'USD', 'BTC', "ftx", {'spot': True}, False),  # Reversed currencies
+    ("BTC/USD", 'BTC', 'USD', "ftx", {'spot': False}, False),  # Can only trade spot markets
+    ("BTC-PERP", 'BTC', 'USD', "ftx", {'spot': False}, False),  # Can only trade spot markets
 ])
-def test_symbol_is_pair(market_symbol, base_currency, quote_currency, expected_result) -> None:
+def test_market_is_tradable(mocker, default_conf, market_symbol, base,
-    assert symbol_is_pair(market_symbol, base_currency, quote_currency) == expected_result
+                            quote, add_dict, exchange, expected_result) -> None:
+    ex = get_patched_exchange(mocker, default_conf, id=exchange)
+    market = {
+        'symbol': market_symbol,
+        'base': base,
+        'quote': quote,
+        **(add_dict),
+    }
+    assert ex.market_is_tradable(market) == expected_result
 
 
 @pytest.mark.parametrize("market,expected_result", [
@@ -2271,3 +2325,27 @@ def test_calculate_fee_rate(mocker, default_conf, order, expected) -> None:
 
     ex = get_patched_exchange(mocker, default_conf)
     assert ex.calculate_fee_rate(order) == expected
+
+
+@pytest.mark.parametrize('retrycount,max_retries,expected', [
+    (0, 3, 10),
+    (1, 3, 5),
+    (2, 3, 2),
+    (3, 3, 1),
+    (0, 1, 2),
+    (1, 1, 1),
+    (0, 4, 17),
+    (1, 4, 10),
+    (2, 4, 5),
+    (3, 4, 2),
+    (4, 4, 1),
+    (0, 5, 26),
+    (1, 5, 17),
+    (2, 5, 10),
+    (3, 5, 5),
+    (4, 5, 2),
+    (5, 5, 1),
+
+])
+def test_calculate_backoff(retrycount, max_retries, expected):
+    assert calculate_backoff(retrycount, max_retries) == expected

tests/exchange/test_ftx.py

@@ -6,9 +6,9 @@ from unittest.mock import MagicMock
 import ccxt
 import pytest
 
-from freqtrade.exceptions import (DependencyException, InvalidOrderException,
+from freqtrade.exceptions import DependencyException, InvalidOrderException
-                                  OperationalException, TemporaryError)
 from tests.conftest import get_patched_exchange
+
 from .test_exchange import ccxt_exceptionhandlers
 
 STOPLOSS_ORDERTYPE = 'stop'
@@ -85,15 +85,9 @@ def test_stoploss_order_ftx(default_conf, mocker):
         exchange = get_patched_exchange(mocker, default_conf, api_mock, 'ftx')
         exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
 
-    with pytest.raises(TemporaryError):
+    ccxt_exceptionhandlers(mocker, default_conf, api_mock, "ftx",
-        api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection"))
+                           "stoploss", "create_order", retries=1,
-        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'ftx')
+                           pair='ETH/BTC', amount=1, stop_price=220, order_types={})
-        exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
-
-    with pytest.raises(OperationalException, match=r".*DeadBeef.*"):
-        api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef"))
-        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'ftx')
-        exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
 
 
 def test_stoploss_order_dry_run_ftx(default_conf, mocker):
@@ -130,34 +124,35 @@ def test_stoploss_adjust_ftx(mocker, default_conf):
     assert not exchange.stoploss_adjust(1501, order)
 
 
-def test_get_stoploss_order(default_conf, mocker):
+def test_fetch_stoploss_order(default_conf, mocker):
     default_conf['dry_run'] = True
     order = MagicMock()
     order.myid = 123
     exchange = get_patched_exchange(mocker, default_conf, id='ftx')
     exchange._dry_run_open_orders['X'] = order
-    assert exchange.get_stoploss_order('X', 'TKN/BTC').myid == 123
+    assert exchange.fetch_stoploss_order('X', 'TKN/BTC').myid == 123
 
     with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'):
-        exchange.get_stoploss_order('Y', 'TKN/BTC')
+        exchange.fetch_stoploss_order('Y', 'TKN/BTC')
 
     default_conf['dry_run'] = False
     api_mock = MagicMock()
     api_mock.fetch_orders = MagicMock(return_value=[{'id': 'X', 'status': '456'}])
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id='ftx')
-    assert exchange.get_stoploss_order('X', 'TKN/BTC')['status'] == '456'
+    assert exchange.fetch_stoploss_order('X', 'TKN/BTC')['status'] == '456'
 
     api_mock.fetch_orders = MagicMock(return_value=[{'id': 'Y', 'status': '456'}])
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id='ftx')
     with pytest.raises(InvalidOrderException, match=r"Could not get stoploss order for id X"):
-        exchange.get_stoploss_order('X', 'TKN/BTC')['status']
+        exchange.fetch_stoploss_order('X', 'TKN/BTC')['status']
 
     with pytest.raises(InvalidOrderException):
         api_mock.fetch_orders = MagicMock(side_effect=ccxt.InvalidOrder("Order not found"))
         exchange = get_patched_exchange(mocker, default_conf, api_mock, id='ftx')
-        exchange.get_stoploss_order(order_id='_', pair='TKN/BTC')
+        exchange.fetch_stoploss_order(order_id='_', pair='TKN/BTC')
     assert api_mock.fetch_orders.call_count == 1
 
     ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'ftx',
-                           'get_stoploss_order', 'fetch_orders',
+                           'fetch_stoploss_order', 'fetch_orders',
+                           retries=6,
                            order_id='_', pair='TKN/BTC')

tests/exchange/test_kraken.py

@@ -6,8 +6,7 @@ from unittest.mock import MagicMock
 import ccxt
 import pytest
 
-from freqtrade.exceptions import (DependencyException, InvalidOrderException,
+from freqtrade.exceptions import DependencyException, InvalidOrderException
-                                  OperationalException, TemporaryError)
 from tests.conftest import get_patched_exchange
 from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
@@ -206,15 +205,9 @@ def test_stoploss_order_kraken(default_conf, mocker):
         exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken')
         exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
 
-    with pytest.raises(TemporaryError):
+    ccxt_exceptionhandlers(mocker, default_conf, api_mock, "kraken",
-        api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection"))
+                           "stoploss", "create_order", retries=1,
-        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken')
+                           pair='ETH/BTC', amount=1, stop_price=220, order_types={})
-        exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
-
-    with pytest.raises(OperationalException, match=r".*DeadBeef.*"):
-        api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef"))
-        exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken')
-        exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
 
 
 def test_stoploss_order_dry_run_kraken(default_conf, mocker):

tests/optimize/test_backtest_detail.py

@@ -395,5 +395,5 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
     for c, trade in enumerate(data.trades):
         res = results.iloc[c]
         assert res.sell_reason == trade.sell_reason
-        assert res.open_time == _get_frame_time_from_offset(trade.open_tick)
+        assert res.open_date == _get_frame_time_from_offset(trade.open_tick)
-        assert res.close_time == _get_frame_time_from_offset(trade.close_tick)
+        assert res.close_date == _get_frame_time_from_offset(trade.close_tick)

tests/optimize/test_backtesting.py

@@ -308,6 +308,11 @@ def test_data_with_fee(default_conf, mocker, testdatadir) -> None:
     assert backtesting.fee == 0.1234
     assert fee_mock.call_count == 0
 
+    default_conf['fee'] = 0.0
+    backtesting = Backtesting(default_conf)
+    assert backtesting.fee == 0.0
+    assert fee_mock.call_count == 0
+
 
 def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None:
     patch_exchange(mocker)
@@ -349,8 +354,8 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
     exists = [
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Backtesting with data from 2017-11-14T21:17:00+00:00 '
+        'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14T22:59:00+00:00 (0 days)..'
+        'up to 2017-11-14 22:59:00 (0 days)..'
     ]
     for line in exists:
         assert log_has(line, caplog)
@@ -459,28 +464,29 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
         {'pair': [pair, pair],
          'profit_percent': [0.0, 0.0],
          'profit_abs': [0.0, 0.0],
-         'open_time': pd.to_datetime([Arrow(2018, 1, 29, 18, 40, 0).datetime,
+         'open_date': pd.to_datetime([Arrow(2018, 1, 29, 18, 40, 0).datetime,
                                       Arrow(2018, 1, 30, 3, 30, 0).datetime], utc=True
                                      ),
-         'close_time': pd.to_datetime([Arrow(2018, 1, 29, 22, 35, 0).datetime,
+         'open_rate': [0.104445, 0.10302485],
+         'open_fee': [0.0025, 0.0025],
+         'close_date': pd.to_datetime([Arrow(2018, 1, 29, 22, 35, 0).datetime,
                                        Arrow(2018, 1, 30, 4, 10, 0).datetime], utc=True),
-         'open_index': [78, 184],
+         'close_rate': [0.104969, 0.103541],
-         'close_index': [125, 192],
+         'close_fee': [0.0025, 0.0025],
+         'amount': [0.00957442, 0.0097064],
          'trade_duration': [235, 40],
          'open_at_end': [False, False],
-         'open_rate': [0.104445, 0.10302485],
-         'close_rate': [0.104969, 0.103541],
          'sell_reason': [SellType.ROI, SellType.ROI]
          })
     pd.testing.assert_frame_equal(results, expected)
     data_pair = processed[pair]
     for _, t in results.iterrows():
-        ln = data_pair.loc[data_pair["date"] == t["open_time"]]
+        ln = data_pair.loc[data_pair["date"] == t["open_date"]]
         # Check open trade rate alignes to open rate
         assert ln is not None
         assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6)
         # check close trade rate alignes to close rate or is between high and low
-        ln = data_pair.loc[data_pair["date"] == t["close_time"]]
+        ln = data_pair.loc[data_pair["date"] == t["close_date"]]
         assert (round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) or
                 round(ln.iloc[0]["low"], 6) < round(
                 t["close_rate"], 6) < round(ln.iloc[0]["high"], 6))
@@ -672,10 +678,10 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Loading data from 2017-11-14T20:57:00+00:00 '
+        'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days)..',
-        'Backtesting with data from 2017-11-14T21:17:00+00:00 '
+        'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...'
     ]
 
@@ -702,6 +708,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
                           generate_pair_metrics=MagicMock(),
                           generate_sell_reason_stats=sell_reason_mock,
                           generate_strategy_metrics=strat_summary,
+                          generate_daily_stats=MagicMock(),
                           )
     patched_configuration_load_config_file(mocker, default_conf)
 
@@ -735,10 +742,10 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Loading data from 2017-11-14T20:57:00+00:00 '
+        'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days)..',
-        'Backtesting with data from 2017-11-14T21:17:00+00:00 '
+        'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...',
         'Running backtesting for Strategy DefaultStrategy',
         'Running backtesting for Strategy TestStrategyLegacy',
@@ -756,13 +763,11 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
         pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC'],
                       'profit_percent': [0.0, 0.0],
                       'profit_abs': [0.0, 0.0],
-                      'open_time': pd.to_datetime(['2018-01-29 18:40:00',
+                      'open_date': pd.to_datetime(['2018-01-29 18:40:00',
                                                    '2018-01-30 03:30:00', ], utc=True
                                                   ),
-                      'close_time': pd.to_datetime(['2018-01-29 20:45:00',
+                      'close_date': pd.to_datetime(['2018-01-29 20:45:00',
                                                     '2018-01-30 05:35:00', ], utc=True),
-                      'open_index': [78, 184],
-                      'close_index': [125, 192],
                       'trade_duration': [235, 40],
                       'open_at_end': [False, False],
                       'open_rate': [0.104445, 0.10302485],
@@ -772,15 +777,13 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
         pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC', 'ETH/BTC'],
                       'profit_percent': [0.03, 0.01, 0.1],
                       'profit_abs': [0.01, 0.02, 0.2],
-                      'open_time': pd.to_datetime(['2018-01-29 18:40:00',
+                      'open_date': pd.to_datetime(['2018-01-29 18:40:00',
                                                    '2018-01-30 03:30:00',
                                                    '2018-01-30 05:30:00'], utc=True
                                                   ),
-                      'close_time': pd.to_datetime(['2018-01-29 20:45:00',
+                      'close_date': pd.to_datetime(['2018-01-29 20:45:00',
                                                     '2018-01-30 05:35:00',
                                                     '2018-01-30 08:30:00'], utc=True),
-                      'open_index': [78, 184, 185],
-                      'close_index': [125, 224, 205],
                       'trade_duration': [47, 40, 20],
                       'open_at_end': [False, False, False],
                       'open_rate': [0.104445, 0.10302485, 0.122541],
@@ -818,10 +821,10 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
-        'Loading data from 2017-11-14T20:57:00+00:00 '
+        'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days)..',
-        'Backtesting with data from 2017-11-14T21:17:00+00:00 '
+        'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14T22:58:00+00:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days)..',
         'Parameter --enable-position-stacking detected ...',
         'Running backtesting for Strategy DefaultStrategy',
         'Running backtesting for Strategy TestStrategyLegacy',

tests/optimize/test_edge_cli.py

@@ -105,3 +105,17 @@ def test_edge_init_fee(mocker, edge_conf) -> None:
     edge_cli = EdgeCli(edge_conf)
     assert edge_cli.edge.fee == 0.1234
     assert fee_mock.call_count == 0
+
+
+def test_edge_start(mocker, edge_conf) -> None:
+    mock_calculate = mocker.patch('freqtrade.edge.edge_positioning.Edge.calculate',
+                                  return_value=True)
+    table_mock = mocker.patch('freqtrade.optimize.edge_cli.generate_edge_table')
+
+    patch_exchange(mocker)
+    edge_conf['stake_amount'] = 20
+
+    edge_cli = EdgeCli(edge_conf)
+    edge_cli.start()
+    assert mock_calculate.call_count == 1
+    assert table_mock.call_count == 1

tests/optimize/test_hyperopt.py

@@ -3,6 +3,7 @@ import locale
 import logging
 from datetime import datetime
 from pathlib import Path
+from copy import deepcopy
 from typing import Dict, List
 from unittest.mock import MagicMock, PropertyMock
 
@@ -16,7 +17,6 @@ from freqtrade.commands.optimize_commands import (setup_optimize_configuration,
                                                   start_hyperopt)
 from freqtrade.data.history import load_data
 from freqtrade.exceptions import DependencyException, OperationalException
-from freqtrade.optimize.default_hyperopt import DefaultHyperOpt
 from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss
 from freqtrade.optimize.hyperopt import Hyperopt
 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver,
@@ -26,15 +26,28 @@ from freqtrade.strategy.interface import SellType
 from tests.conftest import (get_args, log_has, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
 
+from .hyperopts.default_hyperopt import DefaultHyperOpt
+
 
 @pytest.fixture(scope='function')
-def hyperopt(default_conf, mocker):
+def hyperopt_conf(default_conf):
-    default_conf.update({
+    hyperconf = deepcopy(default_conf)
-        'spaces': ['default'],
+    hyperconf.update({
-        'hyperopt': 'DefaultHyperOpt',
+                         'hyperopt': 'DefaultHyperOpt',
-    })
+                         'hyperopt_path': str(Path(__file__).parent / 'hyperopts'),
+                         'epochs': 1,
+                         'timerange': None,
+                         'spaces': ['default'],
+                         'hyperopt_jobs': 1,
+                         })
+    return hyperconf
+
+
+@pytest.fixture(scope='function')
+def hyperopt(hyperopt_conf, mocker):
+
     patch_exchange(mocker)
-    return Hyperopt(default_conf)
+    return Hyperopt(hyperopt_conf)
 
 
 @pytest.fixture(scope='function')
@@ -46,7 +59,7 @@ def hyperopt_results():
             'profit_abs': [-0.2, 0.4, 0.6],
             'trade_duration': [10, 30, 10],
             'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI],
-            'close_time':
+            'close_date':
             [
                 datetime(2019, 1, 1, 9, 26, 3, 478039),
                 datetime(2019, 2, 1, 9, 26, 3, 478039),
@@ -160,7 +173,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo
     assert log_has('Parameter --print-all detected ...', caplog)
 
 
-def test_setup_hyperopt_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None:
+def test_setup_hyperopt_configuration_unlimited_stake_amount(mocker, default_conf) -> None:
     default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
 
     patched_configuration_load_config_file(mocker, default_conf)
@@ -201,7 +214,7 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None:
     assert hasattr(x, "timeframe")
 
 
-def test_hyperoptresolver_wrongname(mocker, default_conf, caplog) -> None:
+def test_hyperoptresolver_wrongname(default_conf) -> None:
     default_conf.update({'hyperopt': "NonExistingHyperoptClass"})
 
     with pytest.raises(OperationalException, match=r'Impossible to load Hyperopt.*'):
@@ -216,7 +229,7 @@ def test_hyperoptresolver_noname(default_conf):
         HyperOptResolver.load_hyperopt(default_conf)
 
 
-def test_hyperoptlossresolver(mocker, default_conf, caplog) -> None:
+def test_hyperoptlossresolver(mocker, default_conf) -> None:
 
     hl = DefaultHyperOptLoss
     mocker.patch(
@@ -227,14 +240,14 @@ def test_hyperoptlossresolver(mocker, default_conf, caplog) -> None:
     assert hasattr(x, "hyperopt_loss_function")
 
 
-def test_hyperoptlossresolver_wrongname(mocker, default_conf, caplog) -> None:
+def test_hyperoptlossresolver_wrongname(default_conf) -> None:
     default_conf.update({'hyperopt_loss': "NonExistingLossClass"})
 
     with pytest.raises(OperationalException, match=r'Impossible to load HyperoptLoss.*'):
         HyperOptLossResolver.load_hyperoptloss(default_conf)
 
 
-def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None:
+def test_start_not_installed(mocker, default_conf, import_fails) -> None:
     start_mock = MagicMock()
     patched_configuration_load_config_file(mocker, default_conf)
 
@@ -245,6 +258,8 @@ def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None
         'hyperopt',
         '--config', 'config.json',
         '--hyperopt', 'DefaultHyperOpt',
+        '--hyperopt-path',
+        str(Path(__file__).parent / "hyperopts"),
         '--epochs', '5'
     ]
     pargs = get_args(args)
@@ -253,9 +268,9 @@ def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None
         start_hyperopt(pargs)
 
 
-def test_start(mocker, default_conf, caplog) -> None:
+def test_start(mocker, hyperopt_conf, caplog) -> None:
     start_mock = MagicMock()
-    patched_configuration_load_config_file(mocker, default_conf)
+    patched_configuration_load_config_file(mocker, hyperopt_conf)
     mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock)
     patch_exchange(mocker)
 
@@ -272,8 +287,8 @@ def test_start(mocker, default_conf, caplog) -> None:
     assert start_mock.call_count == 1
 
 
-def test_start_no_data(mocker, default_conf, caplog) -> None:
+def test_start_no_data(mocker, hyperopt_conf) -> None:
-    patched_configuration_load_config_file(mocker, default_conf)
+    patched_configuration_load_config_file(mocker, hyperopt_conf)
     mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame))
     mocker.patch(
         'freqtrade.optimize.hyperopt.get_timerange',
@@ -293,9 +308,9 @@ def test_start_no_data(mocker, default_conf, caplog) -> None:
         start_hyperopt(pargs)
 
 
-def test_start_filelock(mocker, default_conf, caplog) -> None:
+def test_start_filelock(mocker, hyperopt_conf, caplog) -> None:
-    start_mock = MagicMock(side_effect=Timeout(Hyperopt.get_lock_filename(default_conf)))
+    start_mock = MagicMock(side_effect=Timeout(Hyperopt.get_lock_filename(hyperopt_conf)))
-    patched_configuration_load_config_file(mocker, default_conf)
+    patched_configuration_load_config_file(mocker, hyperopt_conf)
     mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock)
     patch_exchange(mocker)
 
@@ -519,7 +534,7 @@ def test_roi_table_generation(hyperopt) -> None:
     assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0}
 
 
-def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
+def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
@@ -545,15 +560,9 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
     )
     patch_exchange(mocker)
     # Co-test loading timeframe from strategy
-    del default_conf['timeframe']
+    del hyperopt_conf['timeframe']
-    default_conf.update({'config': 'config.json.example',
-                         'hyperopt': 'DefaultHyperOpt',
-                         'epochs': 1,
-                         'timerange': None,
-                         'spaces': 'default',
-                         'hyperopt_jobs': 1, })
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock()
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 
@@ -569,7 +578,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
     assert hasattr(hyperopt.backtesting.strategy, "advise_sell")
     assert hasattr(hyperopt.backtesting.strategy, "advise_buy")
     assert hasattr(hyperopt, "max_open_trades")
-    assert hyperopt.max_open_trades == default_conf['max_open_trades']
+    assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades']
     assert hasattr(hyperopt, "position_stacking")
 
 
@@ -686,13 +695,36 @@ def test_buy_strategy_generator(hyperopt, testdatadir) -> None:
     assert 1 in result['buy']
 
 
-def test_generate_optimizer(mocker, default_conf) -> None:
+def test_sell_strategy_generator(hyperopt, testdatadir) -> None:
-    default_conf.update({'config': 'config.json.example',
+    data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True)
-                         'hyperopt': 'DefaultHyperOpt',
+    dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data)
-                         'timerange': None,
+    dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'],
-                         'spaces': 'all',
+                                                             {'pair': 'UNITTEST/BTC'})
-                         'hyperopt_min_trades': 1,
+
-                         })
+    populate_sell_trend = hyperopt.custom_hyperopt.sell_strategy_generator(
+        {
+            'sell-adx-value': 20,
+            'sell-fastd-value': 75,
+            'sell-mfi-value': 80,
+            'sell-rsi-value': 20,
+            'sell-adx-enabled': True,
+            'sell-fastd-enabled': True,
+            'sell-mfi-enabled': True,
+            'sell-rsi-enabled': True,
+            'sell-trigger': 'sell-bb_upper'
+        }
+    )
+    result = populate_sell_trend(dataframe, {'pair': 'UNITTEST/BTC'})
+    # Check if some indicators are generated. We will not test all of them
+    print(result)
+    assert 'sell' in result
+    assert 1 in result['sell']
+
+
+def test_generate_optimizer(mocker, hyperopt_conf) -> None:
+    hyperopt_conf.update({'spaces': 'all',
+                          'hyperopt_min_trades': 1,
+                          })
 
     trades = [
         ('TRX/BTC', 0.023117, 0.000233, 100)
@@ -744,8 +776,10 @@ def test_generate_optimizer(mocker, default_conf) -> None:
     }
     response_expected = {
         'loss': 1.9840569076926293,
-        'results_explanation': ('     1 trades. Avg profit   2.31%. Total profit  0.00023300 BTC '
+        'results_explanation': ('     1 trades. 1/0/0 Wins/Draws/Losses. '
-                                '(   2.31\N{GREEK CAPITAL LETTER SIGMA}%). Avg duration 100.0 min.'
+                                'Avg profit   2.31%. Median profit   2.31%. Total profit  '
+                                '0.00023300 BTC (   2.31\N{GREEK CAPITAL LETTER SIGMA}%). '
+                                'Avg duration 100.0 min.'
                                 ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8'),
         'params_details': {'buy': {'adx-enabled': False,
                                    'adx-value': 0,
@@ -776,55 +810,47 @@ def test_generate_optimizer(mocker, default_conf) -> None:
                                         'trailing_stop_positive_offset': 0.07}},
         'params_dict': optimizer_param,
         'results_metrics': {'avg_profit': 2.3117,
+                            'draws': 0,
                             'duration': 100.0,
+                            'losses': 0,
+                            'winsdrawslosses': '1/0/0',
+                            'median_profit': 2.3117,
                             'profit': 2.3117,
                             'total_profit': 0.000233,
-                            'trade_count': 1},
+                            'trade_count': 1,
+                            'wins': 1},
         'total_profit': 0.00023300
     }
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.dimensions = hyperopt.hyperopt_space()
     generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values()))
     assert generate_optimizer_value == response_expected
 
 
-def test_clean_hyperopt(mocker, default_conf, caplog):
+def test_clean_hyperopt(mocker, hyperopt_conf, caplog):
     patch_exchange(mocker)
-    default_conf.update({'config': 'config.json.example',
+
-                         'hyperopt': 'DefaultHyperOpt',
-                         'epochs': 1,
-                         'timerange': None,
-                         'spaces': 'default',
-                         'hyperopt_jobs': 1,
-                         })
     mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True))
     unlinkmock = mocker.patch("freqtrade.optimize.hyperopt.Path.unlink", MagicMock())
-    h = Hyperopt(default_conf)
+    h = Hyperopt(hyperopt_conf)
 
     assert unlinkmock.call_count == 2
     assert log_has(f"Removing `{h.data_pickle_file}`.", caplog)
 
 
-def test_continue_hyperopt(mocker, default_conf, caplog):
+def test_continue_hyperopt(mocker, hyperopt_conf, caplog):
     patch_exchange(mocker)
-    default_conf.update({'config': 'config.json.example',
+    hyperopt_conf.update({'hyperopt_continue': True})
-                         'hyperopt': 'DefaultHyperOpt',
-                         'epochs': 1,
-                         'timerange': None,
-                         'spaces': 'default',
-                         'hyperopt_jobs': 1,
-                         'hyperopt_continue': True
-                         })
     mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True))
     unlinkmock = mocker.patch("freqtrade.optimize.hyperopt.Path.unlink", MagicMock())
-    Hyperopt(default_conf)
+    Hyperopt(hyperopt_conf)
 
     assert unlinkmock.call_count == 0
     assert log_has("Continuing on previous hyperopt results.", caplog)
 
 
-def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
+def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
@@ -855,16 +881,12 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
     )
     patch_exchange(mocker)
 
-    default_conf.update({'config': 'config.json.example',
+    hyperopt_conf.update({'spaces': 'all',
-                         'hyperopt': 'DefaultHyperOpt',
+                          'hyperopt_jobs': 1,
-                         'epochs': 1,
+                          'print_json': True,
-                         'timerange': None,
+                          })
-                         'spaces': 'all',
-                         'hyperopt_jobs': 1,
-                         'print_json': True,
-                         })
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock()
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 
@@ -883,7 +905,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
     assert dumper.call_count == 2
 
 
-def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None:
+def test_print_json_spaces_default(mocker, hyperopt_conf, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
@@ -913,16 +935,9 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None
     )
     patch_exchange(mocker)
 
-    default_conf.update({'config': 'config.json.example',
+    hyperopt_conf.update({'print_json': True})
-                         'hyperopt': 'DefaultHyperOpt',
-                         'epochs': 1,
-                         'timerange': None,
-                         'spaces': 'default',
-                         'hyperopt_jobs': 1,
-                         'print_json': True,
-                         })
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock()
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 
@@ -937,7 +952,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None
     assert dumper.call_count == 2
 
 
-def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> None:
+def test_print_json_spaces_roi_stoploss(mocker, hyperopt_conf, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
@@ -963,16 +978,12 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) ->
     )
     patch_exchange(mocker)
 
-    default_conf.update({'config': 'config.json.example',
+    hyperopt_conf.update({'spaces': 'roi stoploss',
-                         'hyperopt': 'DefaultHyperOpt',
+                          'hyperopt_jobs': 1,
-                         'epochs': 1,
+                          'print_json': True,
-                         'timerange': None,
+                          })
-                         'spaces': 'roi stoploss',
-                         'hyperopt_jobs': 1,
-                         'print_json': True,
-                         })
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock()
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 
@@ -987,7 +998,7 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) ->
     assert dumper.call_count == 2
 
 
-def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) -> None:
+def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
@@ -1012,14 +1023,9 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys)
     )
     patch_exchange(mocker)
 
-    default_conf.update({'config': 'config.json.example',
+    hyperopt_conf.update({'spaces': 'roi stoploss'})
-                         'hyperopt': 'DefaultHyperOpt',
-                         'epochs': 1,
-                         'timerange': None,
-                         'spaces': 'roi stoploss',
-                         'hyperopt_jobs': 1, })
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock()
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 
@@ -1040,11 +1046,11 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys)
     assert hasattr(hyperopt.backtesting.strategy, "advise_sell")
     assert hasattr(hyperopt.backtesting.strategy, "advise_buy")
     assert hasattr(hyperopt, "max_open_trades")
-    assert hyperopt.max_open_trades == default_conf['max_open_trades']
+    assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades']
     assert hasattr(hyperopt, "position_stacking")
 
 
-def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -> None:
+def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None:
     mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
@@ -1055,14 +1061,9 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -
 
     patch_exchange(mocker)
 
-    default_conf.update({'config': 'config.json.example',
+    hyperopt_conf.update({'spaces': 'all', })
-                         'hyperopt': 'DefaultHyperOpt',
-                         'epochs': 1,
-                         'timerange': None,
-                         'spaces': 'all',
-                         'hyperopt_jobs': 1, })
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock()
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 
@@ -1075,7 +1076,7 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -
         hyperopt.start()
 
 
-def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
+def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
@@ -1100,14 +1101,9 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
     )
     patch_exchange(mocker)
 
-    default_conf.update({'config': 'config.json.example',
+    hyperopt_conf.update({'spaces': 'buy'})
-                         'hyperopt': 'DefaultHyperOpt',
-                         'epochs': 1,
-                         'timerange': None,
-                         'spaces': 'buy',
-                         'hyperopt_jobs': 1, })
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock()
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 
@@ -1128,11 +1124,11 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None:
     assert hasattr(hyperopt.backtesting.strategy, "advise_sell")
     assert hasattr(hyperopt.backtesting.strategy, "advise_buy")
     assert hasattr(hyperopt, "max_open_trades")
-    assert hyperopt.max_open_trades == default_conf['max_open_trades']
+    assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades']
     assert hasattr(hyperopt, "position_stacking")
 
 
-def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None:
+def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
@@ -1157,14 +1153,9 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None
     )
     patch_exchange(mocker)
 
-    default_conf.update({'config': 'config.json.example',
+    hyperopt_conf.update({'spaces': 'sell', })
-                         'hyperopt': 'DefaultHyperOpt',
-                         'epochs': 1,
-                         'timerange': None,
-                         'spaces': 'sell',
-                         'hyperopt_jobs': 1, })
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock()
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 
@@ -1185,7 +1176,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None
     assert hasattr(hyperopt.backtesting.strategy, "advise_sell")
     assert hasattr(hyperopt.backtesting.strategy, "advise_buy")
     assert hasattr(hyperopt, "max_open_trades")
-    assert hyperopt.max_open_trades == default_conf['max_open_trades']
+    assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades']
     assert hasattr(hyperopt, "position_stacking")
 
 
@@ -1195,7 +1186,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None
     ('sell_strategy_generator', 'sell'),
     ('sell_indicator_space', 'sell'),
 ])
-def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, method, space) -> None:
+def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> None:
     mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
                  MagicMock(return_value=(MagicMock(), None)))
@@ -1206,14 +1197,9 @@ def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, metho
 
     patch_exchange(mocker)
 
-    default_conf.update({'config': 'config.json.example',
+    hyperopt_conf.update({'spaces': space})
-                         'hyperopt': 'DefaultHyperOpt',
-                         'epochs': 1,
-                         'timerange': None,
-                         'spaces': space,
-                         'hyperopt_jobs': 1, })
 
-    hyperopt = Hyperopt(default_conf)
+    hyperopt = Hyperopt(hyperopt_conf)
     hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock()
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 

tests/optimize/test_optimize_reports.py

@@ -1,16 +1,29 @@
+import re
+from datetime import timedelta
 from pathlib import Path
 
 import pandas as pd
 import pytest
 from arrow import Arrow
 
+from freqtrade.configuration import TimeRange
+from freqtrade.constants import LAST_BT_RESULT_FN
+from freqtrade.data import history
+from freqtrade.data.btanalysis import (get_latest_backtest_filename,
+                                       load_backtest_data)
 from freqtrade.edge import PairInfo
-from freqtrade.optimize.optimize_reports import (
+from freqtrade.optimize.optimize_reports import (generate_backtest_stats,
-    generate_pair_metrics, generate_edge_table, generate_sell_reason_stats,
+                                                 generate_daily_stats,
-    text_table_bt_results, text_table_sell_reason, generate_strategy_metrics,
+                                                 generate_edge_table,
-    text_table_strategy, store_backtest_result)
+                                                 generate_pair_metrics,
+                                                 generate_sell_reason_stats,
+                                                 generate_strategy_metrics,
+                                                 store_backtest_stats,
+                                                 text_table_bt_results,
+                                                 text_table_sell_reason,
+                                                 text_table_strategy)
 from freqtrade.strategy.interface import SellType
-from tests.conftest import patch_exchange
+from tests.data.test_history import _backup_file, _clean_test_file
 
 
 def test_text_table_bt_results(default_conf, mocker):
@@ -43,6 +56,115 @@ def test_text_table_bt_results(default_conf, mocker):
     assert text_table_bt_results(pair_results, stake_currency='BTC') == result_str
 
 
+def test_generate_backtest_stats(default_conf, testdatadir):
+    results = {'DefStrat': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC",
+                                                  "UNITTEST/BTC", "UNITTEST/BTC"],
+                                         "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780],
+                                         "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
+                                         "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime,
+                                                       Arrow(2017, 11, 14, 21, 36, 00).datetime,
+                                                       Arrow(2017, 11, 14, 22, 12, 00).datetime,
+                                                       Arrow(2017, 11, 14, 22, 44, 00).datetime],
+                                         "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime,
+                                                        Arrow(2017, 11, 14, 22, 10, 00).datetime,
+                                                        Arrow(2017, 11, 14, 22, 43, 00).datetime,
+                                                        Arrow(2017, 11, 14, 22, 58, 00).datetime],
+                                         "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
+                                         "close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
+                                         "trade_duration": [123, 34, 31, 14],
+                                         "open_at_end": [False, False, False, True],
+                                         "sell_reason": [SellType.ROI, SellType.STOP_LOSS,
+                                                         SellType.ROI, SellType.FORCE_SELL]
+                                         })}
+    timerange = TimeRange.parse_timerange('1510688220-1510700340')
+    min_date = Arrow.fromtimestamp(1510688220)
+    max_date = Arrow.fromtimestamp(1510700340)
+    btdata = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
+                               fill_up_missing=True)
+
+    stats = generate_backtest_stats(default_conf, btdata, results, min_date, max_date)
+    assert isinstance(stats, dict)
+    assert 'strategy' in stats
+    assert 'DefStrat' in stats['strategy']
+    assert 'strategy_comparison' in stats
+    strat_stats = stats['strategy']['DefStrat']
+    assert strat_stats['backtest_start'] == min_date.datetime
+    assert strat_stats['backtest_end'] == max_date.datetime
+    assert strat_stats['total_trades'] == len(results['DefStrat'])
+    # Above sample had no loosing trade
+    assert strat_stats['max_drawdown'] == 0.0
+
+    results = {'DefStrat': pd.DataFrame(
+        {"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"],
+         "profit_percent": [0.003312, 0.010801, -0.013803, 0.002780],
+         "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003],
+         "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime,
+                       Arrow(2017, 11, 14, 21, 36, 00).datetime,
+                       Arrow(2017, 11, 14, 22, 12, 00).datetime,
+                       Arrow(2017, 11, 14, 22, 44, 00).datetime],
+         "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime,
+                        Arrow(2017, 11, 14, 22, 10, 00).datetime,
+                        Arrow(2017, 11, 14, 22, 43, 00).datetime,
+                        Arrow(2017, 11, 14, 22, 58, 00).datetime],
+         "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
+         "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217],
+         "trade_duration": [123, 34, 31, 14],
+         "open_at_end": [False, False, False, True],
+         "sell_reason": [SellType.ROI, SellType.STOP_LOSS,
+                         SellType.ROI, SellType.FORCE_SELL]
+         })}
+
+    assert strat_stats['max_drawdown'] == 0.0
+    assert strat_stats['drawdown_start'] == Arrow.fromtimestamp(0).datetime
+    assert strat_stats['drawdown_end'] == Arrow.fromtimestamp(0).datetime
+    assert strat_stats['drawdown_end_ts'] == 0
+    assert strat_stats['drawdown_start_ts'] == 0
+    assert strat_stats['pairlist'] == ['UNITTEST/BTC']
+
+    # Test storing stats
+    filename = Path(testdatadir / 'btresult.json')
+    filename_last = Path(testdatadir / LAST_BT_RESULT_FN)
+    _backup_file(filename_last, copy_file=True)
+    assert not filename.is_file()
+
+    store_backtest_stats(filename, stats)
+
+    # get real Filename (it's btresult-<date>.json)
+    last_fn = get_latest_backtest_filename(filename_last.parent)
+    assert re.match(r"btresult-.*\.json", last_fn)
+
+    filename1 = (testdatadir / last_fn)
+    assert filename1.is_file()
+    content = filename1.read_text()
+    assert 'max_drawdown' in content
+    assert 'strategy' in content
+    assert 'pairlist' in content
+
+    assert filename_last.is_file()
+
+    _clean_test_file(filename_last)
+    filename1.unlink()
+
+
+def test_store_backtest_stats(testdatadir, mocker):
+
+    dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_json')
+
+    store_backtest_stats(testdatadir, {})
+
+    assert dump_mock.call_count == 2
+    assert isinstance(dump_mock.call_args_list[0][0][0], Path)
+    assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir/'backtest-result'))
+
+    dump_mock.reset_mock()
+    filename = testdatadir / 'testresult.json'
+    store_backtest_stats(filename, {})
+    assert dump_mock.call_count == 2
+    assert isinstance(dump_mock.call_args_list[0][0][0], Path)
+    # result will be testdatadir / testresult-<timestamp>.json
+    assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'testresult'))
+
+
 def test_generate_pair_metrics(default_conf, mocker):
 
     results = pd.DataFrame(
@@ -68,6 +190,29 @@ def test_generate_pair_metrics(default_conf, mocker):
         pytest.approx(pair_results[-1]['profit_sum_pct']) == pair_results[-1]['profit_sum'] * 100)
 
 
+def test_generate_daily_stats(testdatadir):
+
+    filename = testdatadir / "backtest-result_new.json"
+    bt_data = load_backtest_data(filename)
+    res = generate_daily_stats(bt_data)
+    assert isinstance(res, dict)
+    assert round(res['backtest_best_day'], 4) == 0.1796
+    assert round(res['backtest_worst_day'], 4) == -0.1468
+    assert res['winning_days'] == 14
+    assert res['draw_days'] == 4
+    assert res['losing_days'] == 3
+    assert res['winner_holding_avg'] == timedelta(seconds=1440)
+    assert res['loser_holding_avg'] == timedelta(days=1, seconds=21420)
+
+    # Select empty dataframe!
+    res = generate_daily_stats(bt_data.loc[bt_data['open_date'] == '2000-01-01', :])
+    assert isinstance(res, dict)
+    assert round(res['backtest_best_day'], 4) == 0.0
+    assert res['winning_days'] == 0
+    assert res['draw_days'] == 0
+    assert res['losing_days'] == 0
+
+
 def test_text_table_sell_reason(default_conf):
 
     results = pd.DataFrame(
@@ -188,77 +333,3 @@ def test_generate_edge_table(edge_conf, mocker):
     assert generate_edge_table(results).count('| ETH/BTC |') == 1
     assert generate_edge_table(results).count(
         '|   Risk Reward Ratio |   Required Risk Reward |   Expectancy |') == 1
-
-
-def test_backtest_record(default_conf, fee, mocker):
-    names = []
-    records = []
-    patch_exchange(mocker)
-    mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
-    mocker.patch(
-        'freqtrade.optimize.optimize_reports.file_dump_json',
-        new=lambda n, r: (names.append(n), records.append(r))
-    )
-
-    results = {'DefStrat': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC",
-                                                  "UNITTEST/BTC", "UNITTEST/BTC"],
-                                         "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780],
-                                         "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
-                                         "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime,
-                                                       Arrow(2017, 11, 14, 21, 36, 00).datetime,
-                                                       Arrow(2017, 11, 14, 22, 12, 00).datetime,
-                                                       Arrow(2017, 11, 14, 22, 44, 00).datetime],
-                                         "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime,
-                                                        Arrow(2017, 11, 14, 22, 10, 00).datetime,
-                                                        Arrow(2017, 11, 14, 22, 43, 00).datetime,
-                                                        Arrow(2017, 11, 14, 22, 58, 00).datetime],
-                                         "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
-                                         "close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
-                                         "open_index": [1, 119, 153, 185],
-                                         "close_index": [118, 151, 184, 199],
-                                         "trade_duration": [123, 34, 31, 14],
-                                         "open_at_end": [False, False, False, True],
-                                         "sell_reason": [SellType.ROI, SellType.STOP_LOSS,
-                                                         SellType.ROI, SellType.FORCE_SELL]
-                                         })}
-    store_backtest_result(Path("backtest-result.json"), results)
-    # Assert file_dump_json was only called once
-    assert names == [Path('backtest-result.json')]
-    records = records[0]
-    # Ensure records are of correct type
-    assert len(records) == 4
-
-    # reset test to test with strategy name
-    names = []
-    records = []
-    results['Strat'] = results['DefStrat']
-    results['Strat2'] = results['DefStrat']
-    store_backtest_result(Path("backtest-result.json"), results)
-    assert names == [
-        Path('backtest-result-DefStrat.json'),
-        Path('backtest-result-Strat.json'),
-        Path('backtest-result-Strat2.json'),
-    ]
-    records = records[0]
-    # Ensure records are of correct type
-    assert len(records) == 4
-
-    # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117)
-    # Below follows just a typecheck of the schema/type of trade-records
-    oix = None
-    for (pair, profit, date_buy, date_sell, buy_index, dur,
-         openr, closer, open_at_end, sell_reason) in records:
-        assert pair == 'UNITTEST/BTC'
-        assert isinstance(profit, float)
-        # FIX: buy/sell should be converted to ints
-        assert isinstance(date_buy, float)
-        assert isinstance(date_sell, float)
-        assert isinstance(openr, float)
-        assert isinstance(closer, float)
-        assert isinstance(open_at_end, bool)
-        assert isinstance(sell_reason, str)
-        isinstance(buy_index, pd._libs.tslib.Timestamp)
-        if oix:
-            assert buy_index > oix
-        oix = buy_index
-        assert dur > 0

tests/pairlist/test_pairlist.py

@@ -235,7 +235,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}],
      "BTC",  ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']),
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
-     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']),
+     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']),
     # No pair for ETH, VolumePairList
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
      "ETH", []),
@@ -275,11 +275,16 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "PriceFilter", "low_price_ratio": 0.03}],
      "USDT", ['ETH/USDT', 'NANO/USDT']),
-    # Hot is removed by precision_filter, Fuel by low_price_filter.
+    # Hot is removed by precision_filter, Fuel by low_price_ratio, Ripple by min_price.
     ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"},
       {"method": "PrecisionFilter"},
-      {"method": "PriceFilter", "low_price_ratio": 0.02}],
+      {"method": "PriceFilter", "low_price_ratio": 0.02, "min_price": 0.01}],
-     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']),
+    # Hot is removed by precision_filter, Fuel by low_price_ratio, Ethereum by max_price.
+    ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"},
+      {"method": "PrecisionFilter"},
+      {"method": "PriceFilter", "low_price_ratio": 0.02, "max_price": 0.05}],
+     "BTC", ['TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
     # HOT and XRP are removed because below 1250 quoteVolume
     ([{"method": "VolumePairList", "number_assets": 5,
        "sort_key": "quoteVolume", "min_value": 1250}],
@@ -298,11 +303,11 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     # ShuffleFilter
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "ShuffleFilter", "seed": 77}],
-     "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT']),
+     "USDT", ['ADADOUBLE/USDT', 'ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']),
     # ShuffleFilter, other seed
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "ShuffleFilter", "seed": 42}],
-     "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT']),
+     "USDT", ['ADAHALF/USDT', 'NANO/USDT', 'ADADOUBLE/USDT', 'ETH/USDT']),
     # ShuffleFilter, no seed
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "ShuffleFilter"}],
@@ -319,7 +324,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
      "BTC", 'filter_at_the_beginning'),  # OperationalException expected
     # PriceFilter after StaticPairList
     ([{"method": "StaticPairList"},
-      {"method": "PriceFilter", "low_price_ratio": 0.02}],
+      {"method": "PriceFilter", "low_price_ratio": 0.02, "min_price": 0.000001, "max_price": 0.1}],
      "BTC", ['ETH/BTC', 'TKN/BTC']),
     # PriceFilter only
     ([{"method": "PriceFilter", "low_price_ratio": 0.02}],
@@ -342,6 +347,9 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"},
       {"method": "StaticPairList"}],
      "BTC", 'static_in_the_middle'),
+    ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},
+      {"method": "PriceFilter", "low_price_ratio": 0.02}],
+        "USDT", ['ETH/USDT', 'NANO/USDT']),
 ])
 def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
                                       ohlcv_history_list, pairlists, base_currency,
@@ -389,13 +397,17 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
         for pairlist in pairlists:
             if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \
                     len(ohlcv_history_list) <= pairlist['min_days_listed']:
-                assert log_has_re(r'^Removed .* from whitelist, because age is less than '
+                assert log_has_re(r'^Removed .* from whitelist, because age .* is less than '
                                   r'.* day.*', caplog)
             if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
                 assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
                                   r'would be <= stop limit.*', caplog)
             if pairlist['method'] == 'PriceFilter' and whitelist_result:
                 assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or
+                        log_has_re(r'^Removed .* from whitelist, '
+                                   r'because last price < .*%$', caplog) or
+                        log_has_re(r'^Removed .* from whitelist, '
+                                   r'because last price > .*%$', caplog) or
                         log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] "
                                    r"is empty.*", caplog))
             if pairlist['method'] == 'VolumePairList':
@@ -456,7 +468,9 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):
     # BCH/BTC not available
     (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"),
     # BTT/BTC is inactive
-    (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active")
+    (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active"),
+    # XLTCUSDT is not a valid pair
+    (['ETH/BTC', 'TKN/BTC', 'XLTCUSDT'], "is not tradable with Freqtrade"),
 ])
 def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog,
                                        log_message, tickers):
@@ -524,6 +538,37 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
     assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf
 
 
+def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers, caplog):
+    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
+                                 {'method': 'AgeFilter', 'min_days_listed': -1}]
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+
+    with pytest.raises(OperationalException,
+                       match=r'AgeFilter requires min_days_listed to be >= 1'):
+        get_patched_freqtradebot(mocker, default_conf)
+
+
+def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers, caplog):
+    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
+                                 {'method': 'AgeFilter', 'min_days_listed': 99999}]
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+
+    with pytest.raises(OperationalException,
+                       match=r'AgeFilter requires min_days_listed to not exceed '
+                             r'exchange max request size \([0-9]+\)'):
+        get_patched_freqtradebot(mocker, default_conf)
+
+
 def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_history_list):
 
     mocker.patch.multiple('freqtrade.exchange.Exchange',
@@ -547,6 +592,60 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his
     assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count
 
 
+@pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [
+    ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010,
+      "max_price": 1.0},
+     "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below "
+     "0.1% or below 0.00000010 or above 1.00000000.'}]",
+     None
+     ),
+    ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010},
+     "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.1% or below 0.00000010.'}]",
+     None
+     ),
+    ({"method": "PriceFilter", "low_price_ratio": 0.001, "max_price": 1.00010000},
+     "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.1% or above 1.00010000.'}]",
+     None
+     ),
+    ({"method": "PriceFilter", "min_price": 0.00002000},
+     "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.00002000.'}]",
+     None
+     ),
+    ({"method": "PriceFilter"},
+     "[{'PriceFilter': 'PriceFilter - No price filters configured.'}]",
+     None
+     ),
+    ({"method": "PriceFilter", "low_price_ratio": -0.001},
+     None,
+     "PriceFilter requires low_price_ratio to be >= 0"
+     ),  # OperationalException expected
+    ({"method": "PriceFilter", "min_price": -0.00000010},
+     None,
+     "PriceFilter requires min_price to be >= 0"
+     ),  # OperationalException expected
+    ({"method": "PriceFilter", "max_price": -1.00010000},
+     None,
+     "PriceFilter requires max_price to be >= 0"
+     ),  # OperationalException expected
+])
+def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig,
+                          desc_expected, exception_expected):
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True)
+                          )
+    whitelist_conf['pairlists'] = [pairlistconfig]
+
+    if desc_expected is not None:
+        freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+        short_desc = str(freqtrade.pairlists.short_desc())
+        assert short_desc == desc_expected
+    else:  # OperationalException expected
+        with pytest.raises(OperationalException,
+                           match=exception_expected):
+            freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
+
+
 def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog):
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
 

tests/rpc/test_rpc.py

@@ -8,12 +8,13 @@ import pytest
 from numpy import isnan
 
 from freqtrade.edge import PairInfo
-from freqtrade.exceptions import DependencyException, TemporaryError
+from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError
 from freqtrade.persistence import Trade
 from freqtrade.rpc import RPC, RPCException
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
 from freqtrade.state import State
-from tests.conftest import get_patched_freqtradebot, patch_get_signal, create_mock_trades
+from tests.conftest import (create_mock_trades, get_patched_freqtradebot,
+                            patch_get_signal)
 
 
 # Functions for recurrent object patching
@@ -78,7 +79,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'open_rate': 1.098e-05,
         'close_rate': None,
         'current_rate': 1.099e-05,
-        'amount': 91.07468124,
+        'amount': 91.07468123,
+        'amount_requested': 91.07468123,
         'stake_amount': 0.001,
         'close_profit': None,
         'close_profit_pct': None,
@@ -99,6 +101,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'initial_stop_loss_ratio': -0.1,
         'stoploss_current_dist': -1.1080000000000002e-06,
         'stoploss_current_dist_ratio': -0.10081893,
+        'stoploss_current_dist_pct': -10.08,
         'stoploss_entry_dist': -0.00010475,
         'stoploss_entry_dist_ratio': -0.10448878,
         'open_order': None,
@@ -106,7 +109,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
     }
 
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate',
-                 MagicMock(side_effect=DependencyException("Pair 'ETH/BTC' not available")))
+                 MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available")))
     results = rpc._rpc_trade_status()
     assert isnan(results[0]['current_profit'])
     assert isnan(results[0]['current_rate'])
@@ -141,7 +144,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'open_rate': 1.098e-05,
         'close_rate': None,
         'current_rate': ANY,
-        'amount': 91.07468124,
+        'amount': 91.07468123,
+        'amount_requested': 91.07468123,
         'stake_amount': 0.001,
         'close_profit': None,
         'close_profit_pct': None,
@@ -162,6 +166,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'initial_stop_loss_ratio': -0.1,
         'stoploss_current_dist': ANY,
         'stoploss_current_dist_ratio': ANY,
+        'stoploss_current_dist_pct': ANY,
         'stoploss_entry_dist': -0.00010475,
         'stoploss_entry_dist_ratio': -0.10448878,
         'open_order': None,
@@ -209,7 +214,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None:
     assert '-0.41% (-0.06)' == result[0][3]
 
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate',
-                 MagicMock(side_effect=DependencyException("Pair 'ETH/BTC' not available")))
+                 MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available")))
     result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD')
     assert 'instantly' == result[0][2]
     assert 'ETH/BTC' in result[0][1]
@@ -252,11 +257,11 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee,
     assert days['fiat_display_currency'] == default_conf['fiat_display_currency']
     for day in days['data']:
         # [datetime.date(2018, 1, 11), '0.00000000 BTC', '0.000 USD']
-        assert (day['abs_profit'] == '0.00000000' or
+        assert (day['abs_profit'] == 0.0 or
-                day['abs_profit'] == '0.00006217')
+                day['abs_profit'] == 0.00006217)
 
-        assert (day['fiat_value'] == '0.000' or
+        assert (day['fiat_value'] == 0.0 or
-                day['fiat_value'] == '0.767')
+                day['fiat_value'] == 0.76748865)
     # ensure first day is current date
     assert str(days['data'][0]['date']) == str(datetime.utcnow().date())
 
@@ -283,12 +288,66 @@ def test_rpc_trade_history(mocker, default_conf, markets, fee):
     assert isinstance(trades['trades'][1], dict)
 
     trades = rpc._rpc_trade_history(0)
-    assert len(trades['trades']) == 3
+    assert len(trades['trades']) == 2
-    assert trades['trades_count'] == 3
+    assert trades['trades_count'] == 2
-    # The first trade is for ETH ... sorting is descending
+    # The first closed trade is for ETC ... sorting is descending
-    assert trades['trades'][-1]['pair'] == 'ETH/BTC'
+    assert trades['trades'][-1]['pair'] == 'ETC/BTC'
-    assert trades['trades'][0]['pair'] == 'ETC/BTC'
+    assert trades['trades'][0]['pair'] == 'XRP/BTC'
-    assert trades['trades'][1]['pair'] == 'ETC/BTC'
+
+
+def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog):
+    mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
+    stoploss_mock = MagicMock()
+    cancel_mock = MagicMock()
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        markets=PropertyMock(return_value=markets),
+        cancel_order=cancel_mock,
+        cancel_stoploss_order=stoploss_mock,
+    )
+
+    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
+    freqtradebot.strategy.order_types['stoploss_on_exchange'] = True
+    create_mock_trades(fee)
+    rpc = RPC(freqtradebot)
+    with pytest.raises(RPCException, match='invalid argument'):
+        rpc._rpc_delete('200')
+
+    create_mock_trades(fee)
+    trades = Trade.query.all()
+    trades[1].stoploss_order_id = '1234'
+    trades[2].stoploss_order_id = '1234'
+    assert len(trades) > 2
+
+    res = rpc._rpc_delete('1')
+    assert isinstance(res, dict)
+    assert res['result'] == 'success'
+    assert res['trade_id'] == '1'
+    assert res['cancel_order_count'] == 1
+    assert cancel_mock.call_count == 1
+    assert stoploss_mock.call_count == 0
+    cancel_mock.reset_mock()
+    stoploss_mock.reset_mock()
+
+    res = rpc._rpc_delete('2')
+    assert isinstance(res, dict)
+    assert cancel_mock.call_count == 1
+    assert stoploss_mock.call_count == 1
+    assert res['cancel_order_count'] == 2
+
+    stoploss_mock = mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order',
+                                 side_effect=InvalidOrderException)
+
+    res = rpc._rpc_delete('3')
+    assert stoploss_mock.call_count == 1
+    stoploss_mock.reset_mock()
+
+    cancel_mock = mocker.patch('freqtrade.exchange.Exchange.cancel_order',
+                               side_effect=InvalidOrderException)
+
+    res = rpc._rpc_delete('4')
+    assert cancel_mock.call_count == 1
+    assert stoploss_mock.call_count == 0
 
 
 def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee,
@@ -365,7 +424,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee,
 
     # Test non-available pair
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate',
-                 MagicMock(side_effect=DependencyException("Pair 'ETH/BTC' not available")))
+                 MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available")))
     stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency)
     assert stats['trade_count'] == 2
     assert stats['first_trade_date'] == 'just now'
@@ -606,7 +665,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         cancel_order=cancel_order_mock,
-        get_order=MagicMock(
+        fetch_order=MagicMock(
             return_value={
                 'status': 'closed',
                 'type': 'limit',
@@ -652,7 +711,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
     trade = Trade.query.filter(Trade.id == '1').first()
     filled_amount = trade.amount / 2
     mocker.patch(
-        'freqtrade.exchange.Exchange.get_order',
+        'freqtrade.exchange.Exchange.fetch_order',
         return_value={
             'status': 'open',
             'type': 'limit',
@@ -671,7 +730,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
     amount = trade.amount
     # make an limit-buy open trade, if there is no 'filled', don't sell it
     mocker.patch(
-        'freqtrade.exchange.Exchange.get_order',
+        'freqtrade.exchange.Exchange.fetch_order',
         return_value={
             'status': 'open',
             'type': 'limit',
@@ -688,7 +747,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
     freqtradebot.enter_positions()
     # make an limit-sell open trade
     mocker.patch(
-        'freqtrade.exchange.Exchange.get_order',
+        'freqtrade.exchange.Exchange.fetch_order',
         return_value={
             'status': 'open',
             'type': 'limit',

tests/rpc/test_rpc_apiserver.py

@@ -10,10 +10,12 @@ from flask import Flask
 from requests.auth import _basic_auth_str
 
 from freqtrade.__init__ import __version__
+from freqtrade.loggers import setup_logging, setup_logging_pre
 from freqtrade.persistence import Trade
 from freqtrade.rpc.api_server import BASE_URI, ApiServer
 from freqtrade.state import State
-from tests.conftest import get_patched_freqtradebot, log_has, patch_get_signal, create_mock_trades
+from tests.conftest import (create_mock_trades, get_patched_freqtradebot,
+                            log_has, patch_get_signal)
 
 _TEST_USER = "FreqTrader"
 _TEST_PASS = "SuperSecurePassword1!"
@@ -21,6 +23,9 @@ _TEST_PASS = "SuperSecurePassword1!"
 
 @pytest.fixture
 def botclient(default_conf, mocker):
+    setup_logging_pre()
+    setup_logging(default_conf)
+
     default_conf.update({"api_server": {"enabled": True,
                                         "listen_ip_address": "127.0.0.1",
                                         "listen_port": 8080,
@@ -50,6 +55,12 @@ def client_get(client, url):
                                     'Origin': 'http://example.com'})
 
 
+def client_delete(client, url):
+    # Add fake Origin to ensure CORS kicks in
+    return client.delete(url, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS),
+                                       'Origin': 'http://example.com'})
+
+
 def assert_response(response, expected_code=200, needs_cors=True):
     assert response.status_code == expected_code
     assert response.content_type == "application/json"
@@ -81,20 +92,20 @@ def test_api_unauthorized(botclient):
     assert rc.json == {'error': 'Unauthorized'}
 
     # Change only username
-    ftbot.config['api_server']['username'] = "Ftrader"
+    ftbot.config['api_server']['username'] = 'Ftrader'
     rc = client_get(client, f"{BASE_URI}/version")
     assert_response(rc, 401)
     assert rc.json == {'error': 'Unauthorized'}
 
     # Change only password
     ftbot.config['api_server']['username'] = _TEST_USER
-    ftbot.config['api_server']['password'] = "WrongPassword"
+    ftbot.config['api_server']['password'] = 'WrongPassword'
     rc = client_get(client, f"{BASE_URI}/version")
     assert_response(rc, 401)
     assert rc.json == {'error': 'Unauthorized'}
 
-    ftbot.config['api_server']['username'] = "Ftrader"
+    ftbot.config['api_server']['username'] = 'Ftrader'
-    ftbot.config['api_server']['password'] = "WrongPassword"
+    ftbot.config['api_server']['password'] = 'WrongPassword'
 
     rc = client_get(client, f"{BASE_URI}/version")
     assert_response(rc, 401)
@@ -326,6 +337,8 @@ def test_api_show_config(botclient, mocker):
     assert rc.json['exchange'] == 'bittrex'
     assert rc.json['ticker_interval'] == '5m'
     assert rc.json['timeframe'] == '5m'
+    assert rc.json['timeframe_ms'] == 300000
+    assert rc.json['timeframe_min'] == 5
     assert rc.json['state'] == 'running'
     assert not rc.json['trailing_stop']
     assert 'bid_strategy' in rc.json
@@ -350,7 +363,7 @@ def test_api_daily(botclient, mocker, ticker, fee, markets):
     assert rc.json['data'][0]['date'] == str(datetime.utcnow().date())
 
 
-def test_api_trades(botclient, mocker, ticker, fee, markets):
+def test_api_trades(botclient, mocker, fee, markets):
     ftbot, client = botclient
     patch_get_signal(ftbot, (True, False))
     mocker.patch.multiple(
@@ -366,12 +379,81 @@ def test_api_trades(botclient, mocker, ticker, fee, markets):
 
     rc = client_get(client, f"{BASE_URI}/trades")
     assert_response(rc)
-    assert len(rc.json['trades']) == 3
-    assert rc.json['trades_count'] == 3
-    rc = client_get(client, f"{BASE_URI}/trades?limit=2")
-    assert_response(rc)
     assert len(rc.json['trades']) == 2
     assert rc.json['trades_count'] == 2
+    rc = client_get(client, f"{BASE_URI}/trades?limit=1")
+    assert_response(rc)
+    assert len(rc.json['trades']) == 1
+    assert rc.json['trades_count'] == 1
+
+
+def test_api_delete_trade(botclient, mocker, fee, markets):
+    ftbot, client = botclient
+    patch_get_signal(ftbot, (True, False))
+    stoploss_mock = MagicMock()
+    cancel_mock = MagicMock()
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        markets=PropertyMock(return_value=markets),
+        cancel_order=cancel_mock,
+        cancel_stoploss_order=stoploss_mock,
+    )
+    rc = client_delete(client, f"{BASE_URI}/trades/1")
+    # Error - trade won't exist yet.
+    assert_response(rc, 502)
+
+    create_mock_trades(fee)
+    ftbot.strategy.order_types['stoploss_on_exchange'] = True
+    trades = Trade.query.all()
+    trades[1].stoploss_order_id = '1234'
+    assert len(trades) > 2
+
+    rc = client_delete(client, f"{BASE_URI}/trades/1")
+    assert_response(rc)
+    assert rc.json['result_msg'] == 'Deleted trade 1. Closed 1 open orders.'
+    assert len(trades) - 1 == len(Trade.query.all())
+    assert cancel_mock.call_count == 1
+
+    cancel_mock.reset_mock()
+    rc = client_delete(client, f"{BASE_URI}/trades/1")
+    # Trade is gone now.
+    assert_response(rc, 502)
+    assert cancel_mock.call_count == 0
+
+    assert len(trades) - 1 == len(Trade.query.all())
+    rc = client_delete(client, f"{BASE_URI}/trades/2")
+    assert_response(rc)
+    assert rc.json['result_msg'] == 'Deleted trade 2. Closed 2 open orders.'
+    assert len(trades) - 2 == len(Trade.query.all())
+    assert stoploss_mock.call_count == 1
+
+
+def test_api_logs(botclient):
+    ftbot, client = botclient
+    rc = client_get(client, f"{BASE_URI}/logs")
+    assert_response(rc)
+    assert len(rc.json) == 2
+    assert 'logs' in rc.json
+    # Using a fixed comparison here would make this test fail!
+    assert rc.json['log_count'] > 10
+    assert len(rc.json['logs']) == rc.json['log_count']
+
+    assert isinstance(rc.json['logs'][0], list)
+    # date
+    assert isinstance(rc.json['logs'][0][0], str)
+    # created_timestamp
+    assert isinstance(rc.json['logs'][0][1], float)
+    assert isinstance(rc.json['logs'][0][2], str)
+    assert isinstance(rc.json['logs'][0][3], str)
+    assert isinstance(rc.json['logs'][0][4], str)
+
+    rc = client_get(client, f"{BASE_URI}/logs?limit=5")
+    assert_response(rc)
+    assert len(rc.json) == 2
+    assert 'logs' in rc.json
+    # Using a fixed comparison here would make this test fail!
+    assert rc.json['log_count'] == 5
+    assert len(rc.json['logs']) == rc.json['log_count']
 
 
 def test_api_edge_disabled(botclient, mocker, ticker, fee, markets):
@@ -431,14 +513,14 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li
                        'latest_trade_date': 'just now',
                        'latest_trade_timestamp': ANY,
                        'profit_all_coin': 6.217e-05,
-                       'profit_all_fiat': 0,
+                       'profit_all_fiat': 0.76748865,
                        'profit_all_percent': 6.2,
                        'profit_all_percent_mean': 6.2,
                        'profit_all_ratio_mean': 0.06201058,
                        'profit_all_percent_sum': 6.2,
                        'profit_all_ratio_sum': 0.06201058,
                        'profit_closed_coin': 6.217e-05,
-                       'profit_closed_fiat': 0,
+                       'profit_closed_fiat': 0.76748865,
                        'profit_closed_percent': 6.2,
                        'profit_closed_ratio_mean': 0.06201058,
                        'profit_closed_percent_mean': 6.2,
@@ -446,6 +528,8 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li
                        'profit_closed_percent_sum': 6.2,
                        'trade_count': 1,
                        'closed_trade_count': 1,
+                       'winning_trades': 1,
+                       'losing_trades': 0,
                        }
 
 
@@ -515,7 +599,8 @@ def test_api_status(botclient, mocker, ticker, fee, markets):
     rc = client_get(client, f"{BASE_URI}/status")
     assert_response(rc)
     assert len(rc.json) == 1
-    assert rc.json == [{'amount': 91.07468124,
+    assert rc.json == [{'amount': 91.07468123,
+                        'amount_requested': 91.07468123,
                         'base_currency': 'BTC',
                         'close_date': None,
                         'close_date_hum': None,
@@ -548,6 +633,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets):
                         'initial_stop_loss_ratio': -0.1,
                         'stoploss_current_dist': -1.1080000000000002e-06,
                         'stoploss_current_dist_ratio': -0.10081893,
+                        'stoploss_current_dist_pct': -10.08,
                         'stoploss_entry_dist': -0.00010475,
                         'stoploss_entry_dist_ratio': -0.10448878,
                         'trade_id': 1,
@@ -624,7 +710,7 @@ def test_api_forcebuy(botclient, mocker, fee):
     assert rc.json == {"error": "Error querying _forcebuy: Forcebuy not enabled."}
 
     # enable forcebuy
-    ftbot.config["forcebuy_enable"] = True
+    ftbot.config['forcebuy_enable'] = True
 
     fbuy_mock = MagicMock(return_value=None)
     mocker.patch("freqtrade.rpc.RPC._rpc_forcebuy", fbuy_mock)
@@ -637,6 +723,7 @@ def test_api_forcebuy(botclient, mocker, fee):
     fbuy_mock = MagicMock(return_value=Trade(
         pair='ETH/ETH',
         amount=1,
+        amount_requested=1,
         exchange='bittrex',
         stake_amount=1,
         open_rate=0.245441,
@@ -653,6 +740,7 @@ def test_api_forcebuy(botclient, mocker, fee):
                      data='{"pair": "ETH/BTC"}')
     assert_response(rc)
     assert rc.json == {'amount': 1,
+                       'amount_requested': 1,
                        'trade_id': None,
                        'close_date': None,
                        'close_date_hum': None,
@@ -689,7 +777,7 @@ def test_api_forcebuy(botclient, mocker, fee):
                        'min_rate': None,
                        'open_order_id': '123456',
                        'open_rate_requested': None,
-                       'open_trade_price': 0.2460546025,
+                       'open_trade_price': 0.24605460,
                        'sell_reason': None,
                        'sell_order_status': None,
                        'strategy': None,