Merge pull request #2453 from freqtrade/rel/2019-10

Release 2019-10

CONTRIBUTING.md

@@ -114,6 +114,6 @@ Contributors may be given commit privileges. Preference will be given to those w
 1. Access to resources for cross-platform development and testing.
 1. Time to devote to the project regularly.
 
-Beeing a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust FreqTrade with their Exchange API keys).
+Being a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust FreqTrade with their Exchange API keys).
 
-After beeing Committer for some time, a Committer may be named Core Committer and given full repository access.
+After being Committer for some time, a Committer may be named Core Committer and given full repository access.

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7.4-slim-stretch
+FROM python:3.7.5-slim-stretch
 
 RUN apt-get update \
     && apt-get -y install curl build-essential libssl-dev \
@@ -16,9 +16,9 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Install dependencies
-COPY requirements.txt requirements-common.txt /freqtrade/
+COPY requirements.txt requirements-common.txt requirements-hyperopt.txt /freqtrade/
 RUN pip install numpy --no-cache-dir \
-  && pip install -r requirements.txt --no-cache-dir
+  && pip install -r requirements-hyperopt.txt --no-cache-dir
 
 # Install and execute
 COPY . /freqtrade/

config.json.example

@@ -22,7 +22,10 @@
     "ask_strategy":{
         "use_order_book": false,
         "order_book_min": 1,
-        "order_book_max": 9
+        "order_book_max": 9,
+        "use_sell_signal": true,
+        "sell_profit_only": false,
+        "ignore_roi_if_buy_signal": false
     },
     "exchange": {
         "name": "bittrex",
@@ -41,7 +44,7 @@
             "ZEC/BTC",
             "XLM/BTC",
             "NXT/BTC",
-            "POWR/BTC",
+            "TRX/BTC",
             "ADA/BTC",
             "XMR/BTC"
         ],
@@ -49,11 +52,6 @@
             "DOGE/BTC"
         ]
     },
-    "experimental": {
-        "use_sell_signal": false,
-        "sell_profit_only": false,
-        "ignore_roi_if_buy_signal": false
-    },
     "edge": {
         "enabled": false,
         "process_throttle_secs": 3600,

config_binance.json.example

@@ -22,7 +22,10 @@
     "ask_strategy":{
         "use_order_book": false,
         "order_book_min": 1,
-        "order_book_max": 9
+        "order_book_max": 9,
+        "use_sell_signal": true,
+        "sell_profit_only": false,
+        "ignore_roi_if_buy_signal": false
     },
     "exchange": {
         "name": "binance",
@@ -51,11 +54,6 @@
             "BNB/BTC"
         ]
     },
-    "experimental": {
-        "use_sell_signal": false,
-        "sell_profit_only": false,
-        "ignore_roi_if_buy_signal": false
-    },
     "edge": {
         "enabled": false,
         "process_throttle_secs": 3600,

config_full.json.example

@@ -33,7 +33,10 @@
     "ask_strategy":{
         "use_order_book": false,
         "order_book_min": 1,
-        "order_book_max": 9
+        "order_book_max": 9,
+        "use_sell_signal": true,
+        "sell_profit_only": false,
+        "ignore_roi_if_buy_signal": false
     },
     "order_types": {
         "buy": "limit",
@@ -75,7 +78,7 @@
             "ZEC/BTC",
             "XLM/BTC",
             "NXT/BTC",
-            "POWR/BTC",
+            "TRX/BTC",
             "ADA/BTC",
             "XMR/BTC"
         ],
@@ -100,11 +103,6 @@
         "max_trade_duration_minute": 1440,
         "remove_pumps": false
     },
-    "experimental": {
-        "use_sell_signal": false,
-        "sell_profit_only": false,
-        "ignore_roi_if_buy_signal": false
-    },
     "telegram": {
         "enabled": true,
         "token": "your_telegram_token",
@@ -121,7 +119,8 @@
     "initial_state": "running",
     "forcebuy_enable": false,
     "internals": {
-        "process_throttle_secs": 5
+        "process_throttle_secs": 5,
+        "heartbeat_interval": 60
     },
     "strategy": "DefaultStrategy",
     "strategy_path": "user_data/strategies/"

config_kraken.json.example

@@ -22,7 +22,11 @@
     "ask_strategy":{
         "use_order_book": false,
         "order_book_min": 1,
-        "order_book_max": 9
+        "order_book_max": 9,
+        "use_sell_signal": true,
+        "sell_profit_only": false,
+        "ignore_roi_if_buy_signal": false
+
     },
     "exchange": {
         "name": "kraken",
@@ -66,5 +70,6 @@
     "forcebuy_enable": false,
     "internals": {
         "process_throttle_secs": 5
-    }
+    },
+    "download_trades": true
 }

docker-compose.develop.yml

@@ -0,0 +1,20 @@
+---
+version: '3'
+services:
+  freqtrade_develop:
+    build:
+      context: .
+      dockerfile: "./Dockerfile.develop"
+    volumes:
+      - ".:/freqtrade"
+    entrypoint:
+      - "freqtrade"
+
+  freqtrade_bash:
+    build:
+      context: .
+      dockerfile: "./Dockerfile.develop"
+    volumes:
+      - ".:/freqtrade"
+    entrypoint:
+      - "/bin/bash"

docker-compose.yml

@@ -0,0 +1,8 @@
+---
+version: '3'
+services:
+  freqtrade:
+    image: freqtradeorg/freqtrade:master
+    volumes:
+      - "./user_data:/freqtrade/user_data"
+      - "./config.json:/freqtrade/config.json"

docs/backtesting.md

@@ -39,7 +39,7 @@ Assume you downloaded the history data from the Bittrex exchange and kept it in
 You can then use this data for backtesting as follows:
 
 ```bash
-freqtrade backtesting --datadir user_data/data/bittrex-20180101
+freqtrade --datadir user_data/data/bittrex-20180101 backtesting
 ```
 
 #### With a (custom) strategy file
@@ -72,6 +72,17 @@ The exported trades can be used for [further analysis](#further-backtest-result-
 freqtrade backtesting --export trades --export-filename=backtest_samplestrategy.json
 ```
 
+#### Supplying custom fee value
+
+Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt.
+To account for this in backtesting, you can use `--fee 0.001` to supply this value to backtesting.
+This fee must be a percentage, and will be applied twice (once for trade entry, and once for trade exit).
+
+```bash
+freqtrade backtesting --fee 0.001
+```
+
+
 #### Running backtest with smaller testset by using timerange
 
 Use the `--timerange` argument to change how much of the testset you want to use.
@@ -92,12 +103,6 @@ The full timerange specification:
 - Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301`
 - Use tickframes between POSIX timestamps 1527595200 1527618600:
                                                 `--timerange=1527595200-1527618600`
-- Use last 123 tickframes of data: `--timerange=-123`
-- Use first 123 tickframes of data: `--timerange=123-`
-- Use tickframes from line 123 through 456: `--timerange=123-456`
-
-!!! warning
-    Be carefull when using non-date functions - these do not allow you to specify precise dates, so if you updated the test-data it will probably use a different dataset.
 
 ## Understand the backtesting result
 
@@ -184,6 +189,7 @@ Hence, keep in mind that your performance is an integral mix of all different el
 Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions:
 
 - Buys happen at open-price
+- Sell signal sells happen at open-price of the following candle
 - Low happens before high for stoploss, protecting capital first.
 - ROI sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%)
 - Stoploss sells happen exactly at stoploss price, even if low was lower
@@ -192,6 +198,11 @@ Since backtesting lacks some detailed information about what happens within a ca
   - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly)
 - Sell-reason does not explain if a trade was positive or negative, just what triggered the sell (this can look odd if negative ROI values are used)
 
+Taking these assumptions, backtesting tries to mirror real trading as closely as possible. However, backtesting will **never** replace running a strategy in dry-run mode.
+Also, keep in mind that past results don't guarantee future success.
+
+In addition to the above assumptions, strategy authors should carefully read the [Common Mistakes](strategy-customization.md#common-mistakes-when-developing-strategies) section, to avoid using data in backtesting which is not available in real market conditions.
+
 ### Further backtest-result analysis
 
 To further analyze your backtest results, you can [export the trades](#exporting-trades-to-file).

docs/bot-usage.md

@@ -12,17 +12,23 @@ This page explains the different parameters of the bot and how to run it.
 usage: freqtrade [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                  [--userdir PATH] [-s NAME] [--strategy-path PATH]
                  [--db-url PATH] [--sd-notify]
-                 {backtesting,edge,hyperopt,create-userdir,list-exchanges} ...
+                 {backtesting,edge,hyperopt,create-userdir,list-exchanges,list-timeframes,download-data,plot-dataframe,plot-profit}
+                 ...
 
 Free, open source crypto trading bot
 
 positional arguments:
-  {backtesting,edge,hyperopt,create-userdir,list-exchanges}
+  {backtesting,edge,hyperopt,create-userdir,list-exchanges,list-timeframes,download-data,plot-dataframe,plot-profit}
     backtesting         Backtesting module.
     edge                Edge module.
     hyperopt            Hyperopt module.
     create-userdir      Create user-data directory.
     list-exchanges      Print available exchanges.
+    list-timeframes     Print available ticker intervals (timeframes) for the
+                        exchange.
+    download-data       Download backtesting data.
+    plot-dataframe      Plot candles with indicators.
+    plot-profit         Generate plot showing profits.
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -100,7 +106,7 @@ user_data/
 ├── backtest_results
 ├── data
 ├── hyperopts
-├── hyperopts_results
+├── hyperopt_results
 ├── plot
 └── strategies
 ```
@@ -168,22 +174,25 @@ Backtesting also uses the config specified via `-c/--config`.
 
 ```
 usage: freqtrade backtesting [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
-                           [--max_open_trades MAX_OPEN_TRADES]
+                             [--max_open_trades INT]
-                           [--stake_amount STAKE_AMOUNT] [-r] [--eps] [--dmmp]
+                             [--stake_amount STAKE_AMOUNT] [--fee FLOAT]
-                           [-l]
+                             [--eps] [--dmmp]
-                           [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
+                             [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
-                           [--export EXPORT] [--export-filename PATH]
+                             [--export EXPORT] [--export-filename PATH]
 
 optional arguments:
   -h, --help            show this help message and exit
   -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL
-                        Specify ticker interval (1m, 5m, 30m, 1h, 1d).
+                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
+                        `1d`).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
-  --max_open_trades MAX_OPEN_TRADES
+  --max_open_trades INT
                         Specify max_open_trades to use.
   --stake_amount STAKE_AMOUNT
                         Specify stake_amount.
+  --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
+                        entry and exit).
   --eps, --enable-position-stacking
                         Allow buying the same pair multiple times (position
                         stacking).
@@ -193,19 +202,21 @@ optional arguments:
                         number).
   --strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
                         Provide a space-separated list of strategies to
-                        backtest Please note that ticker-interval needs to be
+                        backtest. Please note that ticker-interval needs to be
                         set either in config or via command line. When using
-                        this together with --export trades, the strategy-name
+                        this together with `--export trades`, the strategy-
-                        is injected into the filename (so backtest-data.json
+                        name is injected into the filename (so `backtest-
-                        becomes backtest-data-DefaultStrategy.json
+                        data.json` becomes `backtest-data-
-  --export EXPORT       Export backtest results, argument are: trades. Example
+                        DefaultStrategy.json`
-                        --export=trades
+  --export EXPORT       Export backtest results, argument are: trades.
+                        Example: `--export=trades`
   --export-filename PATH
-                        Save backtest results to this filename requires
+                        Save backtest results to the file with this filename
-                        --export to be set as well Example --export-
+                        (default: `user_data/backtest_results/backtest-
-                        filename=user_data/backtest_results/backtest_today.json
+                        result.json`). Requires `--export` to be set as well.
-                        (default: user_data/backtest_results/backtest-
+                        Example: `--export-filename=user_data/backtest_results
-                        result.json)
+                        /backtest_today.json`
+
 ```
 
 ### Getting historic data for backtesting
@@ -222,13 +233,13 @@ to find optimal parameter values for your stategy.
 ```
 usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
                           [--max_open_trades INT]
-                          [--stake_amount STAKE_AMOUNT] [-r]
+                          [--stake_amount STAKE_AMOUNT] [--fee FLOAT]
                           [--customhyperopt NAME] [--hyperopt-path PATH]
                           [--eps] [-e INT]
                           [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]]
-                          [--dmmp] [--print-all] [--no-color] [-j JOBS]
+                          [--dmmp] [--print-all] [--no-color] [--print-json]
-                          [--random-state INT] [--min-trades INT] [--continue]
+                          [-j JOBS] [--random-state INT] [--min-trades INT]
-                          [--hyperopt-loss NAME]
+                          [--continue] [--hyperopt-loss NAME]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -241,9 +252,11 @@ optional arguments:
                         Specify max_open_trades to use.
   --stake_amount STAKE_AMOUNT
                         Specify stake_amount.
+  --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
+                        entry and exit).
   --customhyperopt NAME
                         Specify hyperopt class name (default:
-                        `DefaultHyperOpts`).
+                        `DefaultHyperOpt`).
   --hyperopt-path PATH  Specify additional lookup path for Hyperopts and
                         Hyperopt Loss functions.
   --eps, --enable-position-stacking
@@ -260,6 +273,7 @@ optional arguments:
   --print-all           Print all results, not only the best ones.
   --no-color            Disable colorization of hyperopt results. May be
                         useful if you are redirecting output to a file.
+  --print-json          Print best result detailization in JSON format.
   -j JOBS, --job-workers JOBS
                         The number of concurrently running jobs for
                         hyperoptimization (hyperopt worker processes). If -1
@@ -278,8 +292,8 @@ optional arguments:
                         generate completely different results, since the
                         target for optimization is different. Built-in
                         Hyperopt-loss-functions are: DefaultHyperOptLoss,
-                        OnlyProfitHyperOptLoss, SharpeHyperOptLoss.
+                        OnlyProfitHyperOptLoss, SharpeHyperOptLoss.(default:
-                        (default: `DefaultHyperOptLoss`).
+                        `DefaultHyperOptLoss`).
 ```
 
 ## Edge commands
@@ -288,25 +302,28 @@ To know your trade expectancy and winrate against historical data, you can use E
 
 ```
 usage: freqtrade edge [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
-                    [--max_open_trades MAX_OPEN_TRADES]
+                      [--max_open_trades INT] [--stake_amount STAKE_AMOUNT]
-                    [--stake_amount STAKE_AMOUNT] [-r]
+                      [--fee FLOAT] [--stoplosses STOPLOSS_RANGE]
-                    [--stoplosses STOPLOSS_RANGE]
 
 optional arguments:
   -h, --help            show this help message and exit
   -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL
-                        Specify ticker interval (1m, 5m, 30m, 1h, 1d).
+                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
+                        `1d`).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
-  --max_open_trades MAX_OPEN_TRADES
+  --max_open_trades INT
                         Specify max_open_trades to use.
   --stake_amount STAKE_AMOUNT
                         Specify stake_amount.
+  --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
+                        entry and exit).
   --stoplosses STOPLOSS_RANGE
-                        Defines a range of stoploss against which edge will
+                        Defines a range of stoploss values against which edge
-                        assess the strategy the format is "min,max,step"
+                        will assess the strategy. The format is "min,max,step"
-                        (without any space).example:
+                        (without any space). Example:
-                        --stoplosses=-0.01,-0.1,-0.001
+                        `--stoplosses=-0.01,-0.1,-0.001`
+
 ```
 
 To understand edge and how to read the results, please read the [edge documentation](edge.md).

docs/configuration.md

@@ -59,12 +59,15 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `unfilledtimeout.sell` | 10 | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled.
 | `bid_strategy.ask_last_balance` | 0.0 | **Required.** Set the bidding price. More information [below](#understand-ask_last_balance).
 | `bid_strategy.use_order_book` | false | Allows buying of pair using the rates in Order Book Bids.
-| `bid_strategy.order_book_top` | 0 | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids.
+| `bid_strategy.order_book_top` | 0 | Bot will use the top N rate in Order Book Bids. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids.
 | `bid_strategy. check_depth_of_market.enabled` | false | Does not buy if the % difference of buy orders and sell orders is met in Order Book.
 | `bid_strategy. check_depth_of_market.bids_to_ask_delta` | 0 | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher.
 | `ask_strategy.use_order_book` | false | Allows selling of open traded pair using the rates in Order Book Asks.
 | `ask_strategy.order_book_min` | 0 | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
 | `ask_strategy.order_book_max` | 0 | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
+| `ask_strategy.use_sell_signal` | true | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
+| `ask_strategy.sell_profit_only` | false | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
+| `ask_strategy.ignore_roi_if_buy_signal` | false | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
 | `order_types` | None | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
 | `order_time_in_force` | None | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
 | `exchange.name` |  | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
@@ -78,9 +81,6 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exchange.ccxt_async_config` | None | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange  and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
 | `exchange.markets_refresh_interval` | 60 | The interval in minutes in which markets are reloaded.
 | `edge` | false | Please refer to [edge configuration document](edge.md) for detailed explanation.
-| `experimental.use_sell_signal` | false | Use your sell strategy in addition of the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
-| `experimental.sell_profit_only` | false | Waits until you have made a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
-| `experimental.ignore_roi_if_buy_signal` | false | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
 | `experimental.block_bad_exchanges` | true | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
 | `pairlist.method` | StaticPairList | Use static or dynamic volume-based pairlist. [More information below](#dynamic-pairlists).
 | `pairlist.config` | None | Additional configuration for dynamic pairlists. [More information below](#dynamic-pairlists).
@@ -98,6 +98,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `strategy` | DefaultStrategy | Defines Strategy class to use.
 | `strategy_path` | null | Adds an additional strategy lookup path (must be a directory).
 | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second.
+| `internals.heartbeat_interval` | 60 | Print heartbeat message every X seconds. Set to 0 to disable heartbeat messages.
 | `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
 | `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file.
 | `user_data_dir` | cwd()/user_data | Directory containing user data. Defaults to `./user_data/`.
@@ -116,9 +117,9 @@ Values set in the configuration file always overwrite values set in the strategy
 * `process_only_new_candles`
 * `order_types`
 * `order_time_in_force`
-* `use_sell_signal` (experimental)
+* `use_sell_signal` (ask_strategy)
-* `sell_profit_only` (experimental)
+* `sell_profit_only` (ask_strategy)
-* `ignore_roi_if_buy_signal` (experimental)
+* `ignore_roi_if_buy_signal` (ask_strategy)
 
 ### Understand stake_amount
 

docs/data-analysis.md

@@ -61,34 +61,6 @@ except:
 print(Path.cwd())
 ```
 
-## Load existing objects into a Jupyter notebook
-
-These examples assume that you have already generated data using the cli. They will allow you to drill deeper into your results, and perform analysis which otherwise would make the output very difficult to digest due to information overload.
-
-### Load backtest results into a pandas dataframe
-
-```python
-from freqtrade.data.btanalysis import load_backtest_data
-
-# Load backtest results
-df = load_backtest_data("user_data/backtest_results/backtest-result.json")
-
-# Show value-counts per pair
-df.groupby("pair")["sell_reason"].value_counts()
-```
-
-### Load live trading results into a pandas dataframe
-
-``` python
-from freqtrade.data.btanalysis import load_trades_from_db
-
-# Fetch trades from database
-df = load_trades_from_db("sqlite:///tradesv3.sqlite")
-
-# Display results
-df.groupby("pair")["sell_reason"].value_counts()
-```
-
 ### Load multiple configuration files
 
 This option can be useful to inspect the results of passing in multiple configs.
@@ -114,99 +86,9 @@ Best avoid relative paths, since this starts at the storage location of the jupy
 }
 ```
 
-### Load exchange data to a pandas dataframe
+### Further Data analysis documentation
 
-This loads candle data to a dataframe
+* [Strategy debugging](strategy_analysis_example.md) - also available as Jupyter notebook (`user_data/notebooks/strategy_analysis_example.ipynb`)
-
+* [Plotting](plotting.md)
-```python
-from pathlib import Path
-from freqtrade.data.history import load_pair_history
-
-# Load data using values passed to function
-ticker_interval = "5m"
-data_location = Path('user_data', 'data', 'bitrex')
-pair = "BTC_USDT"
-candles = load_pair_history(datadir=data_location,
-                            ticker_interval=ticker_interval,
-                            pair=pair)
-
-# Confirm success
-print(f"Loaded len(candles) rows of data for {pair} from {data_location}")
-candles.head()
-```
-
-## Strategy debugging example  
-
-Debugging a strategy can be time-consuming. FreqTrade offers helper functions to visualize raw data.
-
-### Define variables used in analyses  
-
-You can override strategy settings as demonstrated below.
-
-```python
-# Customize these according to your needs.
-
-# Define some constants
-ticker_interval = "5m"
-# Name of the strategy class
-strategy_name = 'SampleStrategy'
-# Path to user data
-user_data_dir = 'user_data'
-# Location of the strategy
-strategy_location = Path(user_data_dir, 'strategies')
-# Location of the data
-data_location = Path(user_data_dir, 'data', 'binance')
-# Pair to analyze - Only use one pair here
-pair = "BTC_USDT"
-```
-
-### Load exchange data
-
-```python
-from pathlib import Path
-from freqtrade.data.history import load_pair_history
-
-# Load data using values set above
-candles = load_pair_history(datadir=data_location,
-                            ticker_interval=ticker_interval,
-                            pair=pair)
-
-# Confirm success
-print(f"Loaded {len(candles)} rows of data for {pair} from {data_location}")
-candles.head()
-```
-
-### Load and run strategy  
-
-* Rerun each time the strategy file is changed
-
-```python
-from freqtrade.resolvers import StrategyResolver
-
-# Load strategy using values set above
-strategy = StrategyResolver({'strategy': strategy_name,
-                            'user_data_dir': user_data_dir,
-                            'strategy_path': strategy_location}).strategy
-
-# Generate buy/sell signals using strategy
-df = strategy.analyze_ticker(candles, {'pair': pair})
-```
-
-### Display the trade details
-
-* Note that using `data.tail()` is preferable to `data.head()` as most indicators have some "startup" data at the top of the dataframe.
-* Some possible problems
-    * Columns with NaN values at the end of the dataframe
-    * Columns used in `crossed*()` functions with completely different units
-* Comparison with full backtest
-    * having 200 buy signals as output for one pair from `analyze_ticker()` does not necessarily mean that 200 trades will be made during backtesting.
-    * Assuming you use only one condition such as, `df['rsi'] < 30` as buy condition, this will generate multiple "buy" signals for each pair in sequence (until rsi returns > 29). The bot will only buy on the first of these signals (and also only if a trade-slot ("max_open_trades") is still available), or on one of the middle signals, as soon as a "slot" becomes available.  
-
-```python
-# Report results
-print(f"Generated {df['buy'].sum()} buy signals")
-data = df.set_index('date', drop=True)
-data.tail()
-```
 
 Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data.

docs/data-download.md

@@ -38,7 +38,7 @@ Mixing different stake-currencies is allowed for this file, since it's only used
 ]
 ```
 
-### start download
+### Start download
 
 Then run:
 
@@ -57,6 +57,32 @@ This will download ticker data for all the currency pairs you defined in `pairs.
 - Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers.
 - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options.
 
+### Trades (tick) data
+
+By default, `download-data` subcommand downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
+This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes.
+
+Since this data is large by default, the files use gzip by default. They are stored in your data-directory with the naming convention of `<pair>-trades.json.gz` (`ETH_BTC-trades.json.gz`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
+
+To use this mode, simply add `--dl-trades` to your call. This will swap the download method to download trades, and resamples the data locally.
+
+Example call:
+
+```bash
+freqtrade download-data --exchange binance --pairs XRP/ETH ETH/BTC --days 20 --dl-trades
+```
+
+!!! Note
+    While this method uses async calls, it will be slow, since it requires the result of the previous call to generate the next request to the exchange.
+
+!!! Warning
+    The historic trades are not available during Freqtrade dry-run and live trade modes because all exchanges tested provide this data with a delay of few 100 candles, so it's not suitable for real-time trading.
+
+### Historic Kraken data
+
+The Kraken API does only provide 720 historic candles, which is sufficient for FreqTrade dry-run and live trade modes, but is a problem for backtesting.
+To download data for the Kraken exchange, using `--dl-trades` is mandatory, otherwise the bot will download the same 720 candles over and over, and you'll not have enough backtest data.
+
 ## Next step
 
 Great, you now have backtest data downloaded, so you can now start [backtesting](backtesting.md) your strategy.

docs/developer.md

@@ -38,8 +38,48 @@ def test_method_to_test(caplog):
     assert log_has("This event happened", caplog)
     # Check regex with trailing number ...
     assert log_has_re(r"This dynamic event happened and produced \d+", caplog)
+
 ```
 
+### Local docker usage
+
+The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine.
+
+#### Install
+* [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+* [docker](https://docs.docker.com/install/)
+* [docker-compose](https://docs.docker.com/compose/install/)
+
+#### Starting the bot
+##### Use the develop dockerfile
+``` bash
+rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml
+```
+#### Docker Compose
+
+##### Starting
+
+``` bash
+docker-compose up
+```
+![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png)
+
+##### Rebuilding
+``` bash
+docker-compose build
+```
+
+##### Execing (effectively SSH into the container)
+
+The `exec` command requires that the container already be running, if you want to start it
+that can be effected by `docker-compose up` or `docker-compose run freqtrade_develop`
+
+``` bash
+docker-compose exec freqtrade_develop /bin/bash
+```
+![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png)
+
+
 ## Modules
 
 ### Dynamic Pairlist
@@ -149,6 +189,15 @@ print(datetime.utcnow())
 The output will show the last entry from the Exchange as well as the current UTC date.
 If the day shows the same day, then the last candle can be assumed as incomplete and should be dropped (leave the setting `"ohlcv_partial_candle"` from the exchange-class untouched / True). Otherwise, set `"ohlcv_partial_candle"` to `False` to not drop Candles (shown in the example above).
 
+## Updating example notebooks
+
+To keep the jupyter notebooks aligned with the documentation, the following should be ran after updating a example notebook.
+
+``` bash
+jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace user_data/notebooks/strategy_analysis_example.ipynb
+jupyter nbconvert --ClearOutputPreprocessor.enabled=True --to markdown user_data/notebooks/strategy_analysis_example.ipynb --stdout > docs/strategy_analysis_example.md
+```
+
 ## Creating a release
 
 This part of the documentation is aimed at maintainers, and shows how to create a release.

docs/edge.md

@@ -249,13 +249,10 @@ freqtrade edge --stoplosses=-0.01,-0.1,-0.001 #min,max,step
 freqtrade edge --timerange=20181110-20181113
 ```
 
-Doing `--timerange=-200` will get the last 200 timeframes from your inputdata. You can also specify specific dates, or a range span indexed by start and stop.
+Doing `--timerange=-20190901` will get all available data until September 1st (excluding September 1st 2019).
 
 The full timerange specification:
 
-* Use last 123 tickframes of data: `--timerange=-123`
-* Use first 123 tickframes of data: `--timerange=123-`
-* Use tickframes from line 123 through 456: `--timerange=123-456`
 * Use tickframes till 2018/01/31: `--timerange=-20180131`
 * Use tickframes since 2018/01/31: `--timerange=20180131-`
 * Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301`

docs/faq.md

@@ -38,7 +38,7 @@ like pauses. You can stop your bot, adjust settings and start it again.
 
 ### I want to improve the bot with a new strategy
 
-That's great. We have a nice backtesting and hyperoptimizing setup. See
+That's great. We have a nice backtesting and hyperoptimization setup. See
 the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-commands).
 
 ### Is there a setting to only SELL the coins being held and not perform anymore BUYS?
@@ -59,7 +59,7 @@ If you're a US customer, the bot will fail to create orders for these pairs, and
 
 ### How many epoch do I need to get a good Hyperopt result?
 
-Per default Hyperopts without `-e` or `--epochs` parameter will only
+Per default Hyperopt called without the `-e`/`--epochs` command line option will only
 run 100 epochs, means 100 evals of your triggers, guards, ... Too few
 to find a great result (unless if you are very lucky), so you probably
 have to run it for 10.000 or more. But it will take an eternity to

docs/hyperopt.md

@@ -10,12 +10,12 @@ Hyperopt requires historic data to be available, just as backtesting does.
 To learn how to get data for the pairs and exchange you're interrested in, head over to the [Data Downloading](data-download.md) section of the documentation.
 
 !!! Bug
-    Hyperopt will crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133)
+    Hyperopt can crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133)
 
 ## Prepare Hyperopting
 
 Before we start digging into Hyperopt, we recommend you to take a look at
-an example hyperopt file located into [user_data/hyperopts/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt.py)
+the sample hyperopt file located in [user_data/hyperopts/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt.py).
 
 Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar and a lot of code can be copied across from the strategy.
 
@@ -64,9 +64,9 @@ multiple guards. The constructed strategy will be something like
 "*buy exactly when close price touches lower bollinger band, BUT only if
 ADX > 10*".
 
-If you have updated the buy strategy, ie. changed the contents of
+If you have updated the buy strategy, i.e. changed the contents of
-`populate_buy_trend()` method you have to update the `guards` and
+`populate_buy_trend()` method, you have to update the `guards` and
-`triggers` hyperopts must use.
+`triggers` your hyperopt must use correspondingly.
 
 #### Sell optimization
 
@@ -82,7 +82,7 @@ To avoid naming collisions in the search-space, please prefix all sell-spaces wi
 #### Using ticker-interval as part of the Strategy
 
 The Strategy exposes the ticker-interval as `self.ticker_interval`. The same value is available as class-attribute `HyperoptName.ticker_interval`.
-In the case of the linked sample-value this would be `SampleHyperOpts.ticker_interval`.
+In the case of the linked sample-value this would be `SampleHyperOpt.ticker_interval`.
 
 ## Solving a Mystery
 

docs/installation.md

@@ -257,14 +257,12 @@ As compiling from source on windows has heavy dependencies (requires a partial v
 ```cmd
 >cd \path\freqtrade-develop
 >python -m venv .env
->cd .env\Scripts
+>.env\Scripts\activate.bat
->activate.bat
->cd \path\freqtrade-develop
 REM optionally install ta-lib from wheel
 REM >pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl
 >pip install -r requirements.txt
 >pip install -e .
->python freqtrade\main.py
+>freqtrade
 ```
 
 > Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222)

docs/partials/header.html

@@ -49,4 +49,6 @@
     </nav>
     <!-- Place this tag in your head or just before your close body tag. -->
     <script async defer src="https://buttons.github.io/buttons.js"></script>
+    <script src="https://code.jquery.com/jquery-3.4.1.min.js"
+        integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
 </header>

docs/plotting.md

@@ -179,5 +179,5 @@ freqtrade plot-profit  -p LTC/BTC --db-url sqlite:///tradesv3.sqlite --trade-sou
 ```
 
 ``` bash
-freqtrade plot-profit --datadir user_data/data/binance_save/ -p LTC/BTC
+freqtrade --datadir user_data/data/binance_save/ plot-profit -p LTC/BTC
 ```

docs/requirements-docs.txt

@@ -1 +1,2 @@
-mkdocs-material==4.4.2
+mkdocs-material==4.4.3
+mdx_truly_sane_lists==1.2

docs/rest-api.md

@@ -100,7 +100,6 @@ python3 scripts/rest_client.py --config rest_config.json <command> [optional par
 | `stopbuy` | | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
 | `reload_conf` | | Reloads the configuration file
 | `status` | | Lists all open trades
-| `status table` | | List all open trades in a table format
 | `count` | | Displays number of trades used and available
 | `profit` | | Display a summary of your profit/loss from close trades and some stats about your performance
 | `forcesell <trade_id>` | | Instantly sells the given trade  (Ignoring `minimum_roi`).

docs/strategy-customization.md

@@ -60,8 +60,7 @@ file as reference.**
 !!! Warning Using future data
     Since backtesting passes the full time interval to the `populate_*()` methods, the strategy author
     needs to take care to avoid having the strategy utilize data from the future.
-    Samples for usage of future data are `dataframe.shift(-1)`, `dataframe.resample("1h")` (this uses the left border of the interval, so moves data from an hour to the start of the hour).
+    Some common patterns for this are listed in the [Common Mistakes](#common-mistakes-when-developing-strategies) section of this document.
-    They all use data which is not available during regular operations, so these strategies will perform well during backtesting, but will fail / perform badly in dry-runs.
 
 ### Customize Indicators
 
@@ -138,15 +137,19 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
     """
     dataframe.loc[
         (
-            (dataframe['adx'] > 30) &
+            (qtpylib.crossed_above(dataframe['rsi'], 30)) &  # Signal: RSI crosses above 30
-            (dataframe['tema'] <= dataframe['bb_middleband']) &
+            (dataframe['tema'] <= dataframe['bb_middleband']) &  # Guard
-            (dataframe['tema'] > dataframe['tema'].shift(1))
+            (dataframe['tema'] > dataframe['tema'].shift(1)) &  # Guard
+            (dataframe['volume'] > 0)  # Make sure Volume is not 0
         ),
         'buy'] = 1
 
     return dataframe
 ```
 
+!!! Note
+    Buying requires sellers to buy from - therefore volume needs to be > 0 (`dataframe['volume'] > 0`) to make sure that the bot does not buy/sell in no-activity periods.
+
 ### Sell signal rules
 
 Edit the method `populate_sell_trend()` into your strategy file to update your sell strategy.
@@ -168,9 +171,10 @@ def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame
     """
     dataframe.loc[
         (
-            (dataframe['adx'] > 70) &
+            (qtpylib.crossed_above(dataframe['rsi'], 70)) &  # Signal: RSI crosses above 70
-            (dataframe['tema'] > dataframe['bb_middleband']) &
+            (dataframe['tema'] > dataframe['bb_middleband']) &  # Guard
-            (dataframe['tema'] < dataframe['tema'].shift(1))
+            (dataframe['tema'] < dataframe['tema'].shift(1)) &  # Guard
+            (dataframe['volume'] > 0)  # Make sure Volume is not 0
         ),
         'sell'] = 1
     return dataframe
@@ -246,9 +250,9 @@ Instead, have a look at the section [Storing information](#Storing-information)
 
 ### Storing information
 
-Storing information can be accomplished by crating a new dictionary within the strategy class.
+Storing information can be accomplished by creating a new dictionary within the strategy class.
 
-The name of the variable can be choosen at will, but should be prefixed with `cust_` to avoid naming collisions with predefined strategy variables.
+The name of the variable can be chosen at will, but should be prefixed with `cust_` to avoid naming collisions with predefined strategy variables.
 
 ```python
 class Awesomestrategy(IStrategy):
@@ -282,6 +286,8 @@ Please always check the mode of operation to select the correct method to get da
 - `ohlcv(pair, ticker_interval)` - Currently cached ticker data for the pair, returns DataFrame or empty DataFrame.
 - `historic_ohlcv(pair, ticker_interval)` - Returns historical data stored on disk.
 - `get_pair_dataframe(pair, ticker_interval)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes).
+- `orderbook(pair, maximum)` - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries.
+- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure.
 - `runmode` - Property containing the current runmode.
 
 #### Example: fetch live ohlcv / historic data for the first informative pair
@@ -344,9 +350,9 @@ def informative_pairs(self):
     As these pairs will be refreshed as part of the regular whitelist refresh, it's best to keep this list short.
     All intervals and all pairs can be specified as long as they are available (and active) on the used exchange.
     It is however better to use resampling to longer time-intervals when possible
-    to avoid hammering the exchange with too many requests and risk beeing blocked.
+    to avoid hammering the exchange with too many requests and risk being blocked.
 
-### Additional data - Wallets
+### Additional data (Wallets)
 
 The strategy provides access to the `Wallets` object. This contains the current balances on the exchange.
 
@@ -392,10 +398,10 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
 
 Printing more than a few rows is also possible (simply use  `print(dataframe)` instead of `print(dataframe.tail())`), however not recommended, as that will be very verbose (~500 lines per pair every 5 seconds).
 
-### Where is the default strategy?
+### Where can i find a strategy template?
 
-The default buy strategy is located in the file
+The strategy template is located in the file
-[freqtrade/default_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/strategy/default_strategy.py).
+[user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/sample_strategy.py).
 
 ### Specify custom strategy location
 
@@ -405,6 +411,18 @@ If you want to use a strategy from a different directory you can pass `--strateg
 freqtrade --strategy AwesomeStrategy --strategy-path /some/directory
 ```
 
+### Common mistakes when developing strategies
+
+Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future.
+This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions.
+
+The following lists some common patterns which should be avoided to prevent frustration:
+
+- don't use `shift(-1)`. This uses data from the future, which is not available.
+- don't use `.iloc[-1]` or any other absolute position in the dataframe, this will be different between dry-run and backtesting.
+- don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling(<window>).mean()` instead
+- don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead.
+
 ### Further strategy ideas
 
 To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk.

docs/strategy_analysis_example.md

@@ -0,0 +1,142 @@
+# Strategy analysis example
+
+Debugging a strategy can be time-consuming. FreqTrade offers helper functions to visualize raw data.
+
+## Setup
+
+
+```python
+from pathlib import Path
+# Customize these according to your needs.
+
+# Define some constants
+ticker_interval = "5m"
+# Name of the strategy class
+strategy_name = 'SampleStrategy'
+# Path to user data
+user_data_dir = Path('user_data')
+# Location of the strategy
+strategy_location = user_data_dir / 'strategies'
+# Location of the data
+data_location = Path(user_data_dir, 'data', 'binance')
+# Pair to analyze - Only use one pair here
+pair = "BTC_USDT"
+```
+
+
+```python
+# Load data using values set above
+from freqtrade.data.history import load_pair_history
+
+candles = load_pair_history(datadir=data_location,
+                            ticker_interval=ticker_interval,
+                            pair=pair)
+
+# Confirm success
+print("Loaded " + str(len(candles)) + f" rows of data for {pair} from {data_location}")
+candles.head()
+```
+
+## Load and run strategy
+* Rerun each time the strategy file is changed
+
+
+```python
+# Load strategy using values set above
+from freqtrade.resolvers import StrategyResolver
+strategy = StrategyResolver({'strategy': strategy_name,
+                            'user_data_dir': user_data_dir,
+                            'strategy_path': strategy_location}).strategy
+
+# Generate buy/sell signals using strategy
+df = strategy.analyze_ticker(candles, {'pair': pair})
+df.tail()
+```
+
+### Display the trade details
+
+* Note that using `data.head()` would also work, however most indicators have some "startup" data at the top of the dataframe.
+* Some possible problems
+    * Columns with NaN values at the end of the dataframe
+    * Columns used in `crossed*()` functions with completely different units
+* Comparison with full backtest
+    * having 200 buy signals as output for one pair from `analyze_ticker()` does not necessarily mean that 200 trades will be made during backtesting.
+    * Assuming you use only one condition such as, `df['rsi'] < 30` as buy condition, this will generate multiple "buy" signals for each pair in sequence (until rsi returns > 29). The bot will only buy on the first of these signals (and also only if a trade-slot ("max_open_trades") is still available), or on one of the middle signals, as soon as a "slot" becomes available.  
+
+
+
+```python
+# Report results
+print(f"Generated {df['buy'].sum()} buy signals")
+data = df.set_index('date', drop=False)
+data.tail()
+```
+
+## Load existing objects into a Jupyter notebook
+
+The following cells assume that you have already generated data using the cli.  
+They will allow you to drill deeper into your results, and perform analysis which otherwise would make the output very difficult to digest due to information overload.
+
+### Load backtest results to pandas dataframe
+
+Analyze a trades dataframe (also used below for plotting)
+
+
+```python
+from freqtrade.data.btanalysis import load_backtest_data
+
+# Load backtest results
+trades = load_backtest_data(user_data_dir / "backtest_results/backtest-result.json")
+
+# Show value-counts per pair
+trades.groupby("pair")["sell_reason"].value_counts()
+```
+
+### Load live trading results into a pandas dataframe
+
+In case you did already some trading and want to analyze your performance
+
+
+```python
+from freqtrade.data.btanalysis import load_trades_from_db
+
+# Fetch trades from database
+trades = load_trades_from_db("sqlite:///tradesv3.sqlite")
+
+# Display results
+trades.groupby("pair")["sell_reason"].value_counts()
+```
+
+## Plot results
+
+Freqtrade offers interactive plotting capabilities based on plotly.
+
+
+```python
+from freqtrade.plot.plotting import  generate_candlestick_graph
+# Limit graph period to keep plotly quick and reactive
+
+data_red = data['2019-06-01':'2019-06-10']
+# Generate candlestick graph
+graph = generate_candlestick_graph(pair=pair,
+                                   data=data_red,
+                                   trades=trades,
+                                   indicators1=['sma20', 'ema50', 'ema55'],
+                                   indicators2=['rsi', 'macd', 'macdsignal', 'macdhist']
+                                  )
+
+
+
+```
+
+
+```python
+# Show graph inline
+# graph.show()
+
+# Render graph in a seperate window
+graph.show(renderer="browser")
+
+```
+
+Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data.

docs/stylesheets/ft.extra.css

@@ -0,0 +1,13 @@
+.rst-versions  {
+    font-size: .7rem;
+    color: white;
+}
+
+.rst-versions.rst-badge .rst-current-version {
+    font-size: .7rem;
+    color: white;
+}
+
+.rst-versions .rst-other-versions {
+    color: white;
+}

docs/utils.md

@@ -0,0 +1,126 @@
+# Utility Subcommands
+
+Besides the Live-Trade and Dry-Run run modes, the `backtesting`, `edge` and `hyperopt` optimization subcommands, and the `download-data` subcommand which prepares historical data, the bot contains a number of utility subcommands. They are described in this section.
+
+## List Exchanges
+
+Use the `list-exchanges` subcommand to see the exchanges available for the bot.
+
+```
+usage: freqtrade list-exchanges [-h] [-1] [-a]
+
+optional arguments:
+  -h, --help        show this help message and exit
+  -1, --one-column  Print output in one column.
+  -a, --all         Print all exchanges known to the ccxt library.
+```
+
+* Example: see exchanges available for the bot:
+```
+$ freqtrade list-exchanges
+Exchanges available for Freqtrade: _1btcxe, acx, allcoin, bequant, bibox, binance, binanceje, binanceus, bitbank, bitfinex, bitfinex2, bitkk, bitlish, bitmart, bittrex, bitz, bleutrade, btcalpha, btcmarkets, btcturk, buda, cex, cobinhood, coinbaseprime, coinbasepro, coinex, cointiger, coss, crex24, digifinex, dsx, dx, ethfinex, fcoin, fcoinjp, gateio, gdax, gemini, hitbtc2, huobipro, huobiru, idex, kkex, kraken, kucoin, kucoin2, kuna, lbank, mandala, mercado, oceanex, okcoincny, okcoinusd, okex, okex3, poloniex, rightbtc, theocean, tidebit, upbit, zb
+```
+
+* Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade):
+```
+$ freqtrade list-exchanges -a
+All exchanges supported by the ccxt library: _1btcxe, acx, adara, allcoin, anxpro, bcex, bequant, bibox, bigone, binance, binanceje, binanceus, bit2c, bitbank, bitbay, bitfinex, bitfinex2, bitflyer, bitforex, bithumb, bitkk, bitlish, bitmart, bitmex, bitso, bitstamp, bitstamp1, bittrex, bitz, bl3p, bleutrade, braziliex, btcalpha, btcbox, btcchina, btcmarkets, btctradeim, btctradeua, btcturk, buda, bxinth, cex, chilebit, cobinhood, coinbase, coinbaseprime, coinbasepro, coincheck, coinegg, coinex, coinexchange, coinfalcon, coinfloor, coingi, coinmarketcap, coinmate, coinone, coinspot, cointiger, coolcoin, coss, crex24, crypton, deribit, digifinex, dsx, dx, ethfinex, exmo, exx, fcoin, fcoinjp, flowbtc, foxbit, fybse, gateio, gdax, gemini, hitbtc, hitbtc2, huobipro, huobiru, ice3x, idex, independentreserve, indodax, itbit, kkex, kraken, kucoin, kucoin2, kuna, lakebtc, latoken, lbank, liquid, livecoin, luno, lykke, mandala, mercado, mixcoins, negociecoins, nova, oceanex, okcoincny, okcoinusd, okex, okex3, paymium, poloniex, rightbtc, southxchange, stronghold, surbitcoin, theocean, therock, tidebit, tidex, upbit, vaultoro, vbtc, virwox, xbtce, yobit, zaif, zb
+```
+
+## List Timeframes
+
+Use the `list-timeframes` subcommand to see the list of ticker intervals (timeframes) available for the exchange.
+
+```
+usage: freqtrade list-timeframes [-h] [--exchange EXCHANGE] [-1]
+
+optional arguments:
+  -h, --help           show this help message and exit
+  --exchange EXCHANGE  Exchange name (default: `bittrex`). Only valid if no
+                       config is provided.
+  -1, --one-column     Print output in one column.
+
+```
+
+* Example: see the timeframes for the 'binance' exchange, set in the configuration file:
+
+```
+$ freqtrade -c config_binance.json list-timeframes
+...
+Timeframes available for the exchange `binance`: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M
+```
+
+* Example: enumerate exchanges available for Freqtrade and print timeframes supported by each of them:
+```
+$ for i in `freqtrade list-exchanges -1`; do freqtrade list-timeframes --exchange $i; done
+```
+
+## List pairs/list markets
+
+The `list-pairs` and `list-markets` subcommands allow to see the pairs/markets available on exchange.
+
+Pairs are markets with the '/' character between the base currency part and the quote currency part in the market symbol.
+For example, in the 'ETH/BTC' pair 'ETH' is the base currency, while 'BTC' is the quote currency.
+
+For pairs traded by Freqtrade the pair quote currency is defined by the value of the `stake_currency` configuration setting.
+
+You can print info about any pair/market with these subcommands - and you can filter output by quote-currency using `--quote BTC`, or by base-currency using `--base ETH` options correspondingly.
+
+These subcommands have same usage and same set of available options:
+
+```
+usage: freqtrade list-markets [-h] [--exchange EXCHANGE] [--print-list]
+                              [--print-json] [-1] [--print-csv]
+                              [--base BASE_CURRENCY [BASE_CURRENCY ...]]
+                              [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]]
+                              [-a]
+
+usage: freqtrade list-pairs [-h] [--exchange EXCHANGE] [--print-list]
+                            [--print-json] [-1] [--print-csv]
+                            [--base BASE_CURRENCY [BASE_CURRENCY ...]]
+                            [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
+                        config is provided.
+  --print-list          Print list of pairs or market symbols. By default data
+                        is printed in the tabular format.
+  --print-json          Print list of pairs or market symbols in JSON format.
+  -1, --one-column      Print output in one column.
+  --print-csv           Print exchange pair or market data in the csv format.
+  --base BASE_CURRENCY [BASE_CURRENCY ...]
+                        Specify base currency(-ies). Space-separated list.
+  --quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]
+                        Specify quote currency(-ies). Space-separated list.
+  -a, --all             Print all pairs or market symbols. By default only
+                        active ones are shown.
+```
+
+By default, only active pairs/markets are shown. Active pairs/markets are those that can currently be traded
+on the exchange. The see the list of all pairs/markets (not only the active ones), use the `-a`/`-all` option.
+
+Pairs/markets are sorted by its symbol string in the printed output.
+
+### Examples
+
+* Print the list of active pairs with quote currency USD on exchange, specified in the default
+configuration file (i.e. pairs on the "Bittrex" exchange) in JSON format:
+
+```
+$ freqtrade list-pairs --quote USD --print-json
+```
+
+* Print the list of all pairs on the exchange, specified in the `config_binance.json` configuration file
+(i.e. on the "Binance" exchange) with base currencies BTC or ETH and quote currencies USDT or USD, as the
+human-readable list with summary:
+
+```
+$ freqtrade -c config_binance.json list-pairs --all --base BTC ETH --quote USDT USD --print-list
+```
+
+* Print all markets on exchange "Kraken", in the tabular format:
+
+```
+$ freqtrade list-markets --exchange kraken --all
+```

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ FreqTrade bot """
-__version__ = '2019.9'
+__version__ = '2019.10'
 
 if __version__ == 'develop':
 

freqtrade/configuration/arguments.py

@@ -2,6 +2,7 @@
 This module contains the argument manager class
 """
 import argparse
+from functools import partial
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
@@ -15,7 +16,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
 ARGS_MAIN = ARGS_COMMON + ARGS_STRATEGY + ["db_url", "sd_notify"]
 
 ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange",
-                        "max_open_trades", "stake_amount"]
+                        "max_open_trades", "stake_amount", "fee"]
 
 ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
                                         "strategy_list", "export", "exportfilename"]
@@ -29,11 +30,17 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
 
 ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
 
-ARGS_LIST_EXCHANGES = ["print_one_column"]
+ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"]
+
+ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"]
+
+ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one_column",
+                   "print_csv", "base_currencies", "quote_currencies", "list_pairs_all"]
 
 ARGS_CREATE_USERDIR = ["user_data_dir"]
 
-ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"]
+ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
+                      "timeframes", "erase"]
 
 ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url",
                        "trade_source", "export", "exportfilename", "timerange", "ticker_interval"]
@@ -41,7 +48,10 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
                     "trade_source", "ticker_interval"]
 
-NO_CONF_REQURIED = ["create-userdir", "download-data", "plot-dataframe", "plot-profit"]
+NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs",
+                    "plot-dataframe", "plot-profit"]
+
+NO_CONF_ALLOWED = ["create-userdir", "list-exchanges"]
 
 
 class Arguments:
@@ -75,12 +85,15 @@ class Arguments:
         parsed_arg = self.parser.parse_args(self.args)
 
         # When no config is provided, but a config exists, use that configuration!
+        subparser = parsed_arg.subparser if 'subparser' in parsed_arg else None
 
         # Workaround issue in argparse with action='append' and default value
         # (see https://bugs.python.org/issue16399)
         # Allow no-config for certain commands (like downloading / plotting)
-        if (parsed_arg.config is None and ((Path.cwd() / constants.DEFAULT_CONFIG).is_file() or
+        if (parsed_arg.config is None
-           not ('subparser' in parsed_arg and parsed_arg.subparser in NO_CONF_REQURIED))):
+                and subparser not in NO_CONF_ALLOWED
+                and ((Path.cwd() / constants.DEFAULT_CONFIG).is_file()
+                     or (subparser not in NO_CONF_REQURIED))):
             parsed_arg.config = [constants.DEFAULT_CONFIG]
 
         return parsed_arg
@@ -98,7 +111,9 @@ class Arguments:
         :return: None
         """
         from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge
-        from freqtrade.utils import start_create_userdir, start_download_data, start_list_exchanges
+        from freqtrade.utils import (start_create_userdir, start_download_data,
+                                     start_list_exchanges, start_list_timeframes,
+                                     start_list_markets)
 
         subparsers = self.parser.add_subparsers(dest='subparser')
 
@@ -131,6 +146,30 @@ class Arguments:
         list_exchanges_cmd.set_defaults(func=start_list_exchanges)
         self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd)
 
+        # Add list-timeframes subcommand
+        list_timeframes_cmd = subparsers.add_parser(
+            'list-timeframes',
+            help='Print available ticker intervals (timeframes) for the exchange.'
+        )
+        list_timeframes_cmd.set_defaults(func=start_list_timeframes)
+        self._build_args(optionlist=ARGS_LIST_TIMEFRAMES, parser=list_timeframes_cmd)
+
+        # Add list-markets subcommand
+        list_markets_cmd = subparsers.add_parser(
+            'list-markets',
+            help='Print markets on exchange.'
+        )
+        list_markets_cmd.set_defaults(func=partial(start_list_markets, pairs_only=False))
+        self._build_args(optionlist=ARGS_LIST_PAIRS, parser=list_markets_cmd)
+
+        # Add list-pairs subcommand
+        list_pairs_cmd = subparsers.add_parser(
+            'list-pairs',
+            help='Print pairs on exchange.'
+        )
+        list_pairs_cmd.set_defaults(func=partial(start_list_markets, pairs_only=True))
+        self._build_args(optionlist=ARGS_LIST_PAIRS, parser=list_pairs_cmd)
+
         # Add download-data subcommand
         download_data_cmd = subparsers.add_parser(
             'download-data',

freqtrade/configuration/check_exchange.py

@@ -3,7 +3,7 @@ from typing import Any, Dict
 
 from freqtrade import OperationalException
 from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason,
-                                is_exchange_available, is_exchange_bad,
+                                is_exchange_known_ccxt, is_exchange_bad,
                                 is_exchange_officially_supported)
 from freqtrade.state import RunMode
 
@@ -31,15 +31,15 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
         raise OperationalException(
             f'This command requires a configured exchange. You should either use '
             f'`--exchange <exchange_name>` or specify a configuration file via `--config`.\n'
-            f'The following exchanges are supported by ccxt: '
+            f'The following exchanges are available for Freqtrade: '
             f'{", ".join(available_exchanges())}'
         )
 
-    if not is_exchange_available(exchange):
+    if not is_exchange_known_ccxt(exchange):
         raise OperationalException(
-                f'Exchange "{exchange}" is not supported by ccxt '
+                f'Exchange "{exchange}" is not known to the ccxt library '
                 f'and therefore not available for the bot.\n'
-                f'The following exchanges are supported by ccxt: '
+                f'The following exchanges are available for Freqtrade: '
                 f'{", ".join(available_exchanges())}'
         )
 
@@ -51,8 +51,8 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
         logger.info(f'Exchange "{exchange}" is officially supported '
                     f'by the Freqtrade development team.')
     else:
-        logger.warning(f'Exchange "{exchange}" is supported by ccxt '
+        logger.warning(f'Exchange "{exchange}" is known to the the ccxt library, '
-                       f'and therefore available for the bot but not officially supported '
+                       f'available for the bot, but not officially supported '
                        f'by the Freqtrade development team. '
                        f'It may work flawlessly (please report back) or have serious issues. '
                        f'Use it at your own discretion.')

freqtrade/configuration/cli_options.py

@@ -2,7 +2,6 @@
 Definition of cli arguments used in arguments.py
 """
 import argparse
-import os
 
 from freqtrade import __version__, constants
 
@@ -141,8 +140,12 @@ AVAILABLE_CLI_OPTIONS = {
         'Requires `--export` to be set as well. '
         'Example: `--export-filename=user_data/backtest_results/backtest_today.json`',
         metavar='PATH',
-        default=os.path.join('user_data', 'backtest_results',
+    ),
-                             'backtest-result.json'),
+    "fee": Arg(
+        '--fee',
+        help='Specify fee ratio. Will be applied twice (on trade entry and exit).',
+        type=float,
+        metavar='FLOAT',
     ),
     # Edge
     "stoploss_range": Arg(
@@ -241,9 +244,50 @@ AVAILABLE_CLI_OPTIONS = {
     # List exchanges
     "print_one_column": Arg(
         '-1', '--one-column',
-        help='Print exchanges in one column.',
+        help='Print output in one column.',
         action='store_true',
     ),
+    "list_exchanges_all": Arg(
+        '-a', '--all',
+        help='Print all exchanges known to the ccxt library.',
+        action='store_true',
+    ),
+    # List pairs / markets
+    "list_pairs_all": Arg(
+        '-a', '--all',
+        help='Print all pairs or market symbols. By default only active '
+             'ones are shown.',
+        action='store_true',
+    ),
+    "print_list": Arg(
+        '--print-list',
+        help='Print list of pairs or market symbols. By default data is '
+             'printed in the tabular format.',
+        action='store_true',
+    ),
+    "list_pairs_print_json": Arg(
+        '--print-json',
+        help='Print list of pairs or market symbols in JSON format.',
+        action='store_true',
+        default=False,
+    ),
+    "print_csv": Arg(
+        '--print-csv',
+        help='Print exchange pair or market data in the csv format.',
+        action='store_true',
+    ),
+    "quote_currencies": Arg(
+        '--quote',
+        help='Specify quote currency(-ies). Space-separated list.',
+        nargs='+',
+        metavar='QUOTE_CURRENCY',
+    ),
+    "base_currencies": Arg(
+        '--base',
+        help='Specify base currency(-ies). Space-separated list.',
+        nargs='+',
+        metavar='BASE_CURRENCY',
+    ),
     # Script options
     "pairs": Arg(
         '-p', '--pairs',
@@ -262,6 +306,12 @@ AVAILABLE_CLI_OPTIONS = {
         type=check_int_positive,
         metavar='INT',
     ),
+    "download_trades": Arg(
+        '--dl-trades',
+        help='Download trades instead of OHLCV data. The bot will resample trades to the '
+             'desired timeframe as specified as --timeframes/-t.',
+        action='store_true',
+    ),
     "exchange": Arg(
         '--exchange',
         help=f'Exchange name (default: `{constants.DEFAULT_EXCHANGE}`). '

freqtrade/configuration/configuration.py

@@ -9,8 +9,9 @@ from typing import Any, Callable, Dict, List, Optional
 
 from freqtrade import OperationalException, constants
 from freqtrade.configuration.check_exchange import check_exchange
-from freqtrade.configuration.config_validation import (
+from freqtrade.configuration.config_validation import (validate_config_consistency,
-    validate_config_consistency, validate_config_schema)
+                                                       validate_config_schema)
+from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
 from freqtrade.configuration.directory_operations import (create_datadir,
                                                           create_userdata_dir)
 from freqtrade.configuration.load_config import load_config_file
@@ -75,6 +76,10 @@ class Configuration:
         # Normalize config
         if 'internals' not in config:
             config['internals'] = {}
+        # TODO: This can be deleted along with removal of deprecated
+        # experimental settings
+        if 'ask_strategy' not in config:
+            config['ask_strategy'] = {}
 
         # validate configuration before returning
         logger.info('Validating configuration ...')
@@ -106,6 +111,8 @@ class Configuration:
 
         self._resolve_pairs_list(config)
 
+        process_temporary_deprecated_settings(config)
+
         validate_config_consistency(config)
 
         return config
@@ -185,6 +192,13 @@ class Configuration:
         config.update({'datadir': create_datadir(config, self.args.get("datadir", None))})
         logger.info('Using data directory: %s ...', config.get('datadir'))
 
+        if self.args.get('exportfilename'):
+            self._args_to_config(config, argname='exportfilename',
+                                 logstring='Storing backtest results to {} ...')
+        else:
+            config['exportfilename'] = (config['user_data_dir']
+                                        / 'backtest_results/backtest-result.json')
+
     def _process_optimize_options(self, config: Dict[str, Any]) -> None:
 
         # This will override the strategy configuration
@@ -210,6 +224,10 @@ class Configuration:
                              logstring='Parameter --stake_amount detected, '
                              'overriding stake_amount to: {} ...')
 
+        self._args_to_config(config, argname='fee',
+                             logstring='Parameter --fee detected, '
+                             'setting fee to: {} ...')
+
         self._args_to_config(config, argname='timerange',
                              logstring='Parameter --timerange detected: {} ...')
 
@@ -224,9 +242,6 @@ class Configuration:
         self._args_to_config(config, argname='export',
                              logstring='Parameter --export detected: {} ...')
 
-        self._args_to_config(config, argname='exportfilename',
-                             logstring='Storing backtest results to {} ...')
-
         # Edge section:
         if 'stoploss_range' in self.args and self.args["stoploss_range"]:
             txt_range = eval(self.args["stoploss_range"])
@@ -301,6 +316,8 @@ class Configuration:
 
         self._args_to_config(config, argname='days',
                              logstring='Detected --days: {}')
+        self._args_to_config(config, argname='download_trades',
+                             logstring='Detected --dl-trades: {}')
 
     def _process_runmode(self, config: Dict[str, Any]) -> None:
 
@@ -323,7 +340,8 @@ class Configuration:
                         sample: logfun=len (prints the length of the found
                         configuration instead of the content)
         """
-        if argname in self.args and self.args[argname]:
+        if (argname in self.args and self.args[argname] is not None
+           and self.args[argname] is not False):
 
             config.update({argname: self.args[argname]})
             if logfun:

freqtrade/configuration/deprecated_settings.py

@@ -0,0 +1,59 @@
+"""
+Functions to handle deprecated settings
+"""
+
+import logging
+from typing import Any, Dict
+
+from freqtrade import OperationalException
+
+
+logger = logging.getLogger(__name__)
+
+
+def check_conflicting_settings(config: Dict[str, Any],
+                               section1: str, name1: str,
+                               section2: str, name2: str):
+    section1_config = config.get(section1, {})
+    section2_config = config.get(section2, {})
+    if name1 in section1_config and name2 in section2_config:
+        raise OperationalException(
+            f"Conflicting settings `{section1}.{name1}` and `{section2}.{name2}` "
+            "(DEPRECATED) detected in the configuration file. "
+            "This deprecated setting will be removed in the next versions of Freqtrade. "
+            f"Please delete it from your configuration and use the `{section1}.{name1}` "
+            "setting instead."
+        )
+
+
+def process_deprecated_setting(config: Dict[str, Any],
+                               section1: str, name1: str,
+                               section2: str, name2: str):
+    section2_config = config.get(section2, {})
+
+    if name2 in section2_config:
+        logger.warning(
+            "DEPRECATED: "
+            f"The `{section2}.{name2}` setting is deprecated and "
+            "will be removed in the next versions of Freqtrade. "
+            f"Please use the `{section1}.{name1}` setting in your configuration instead."
+        )
+        section1_config = config.get(section1, {})
+        section1_config[name1] = section2_config[name2]
+
+
+def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
+
+    check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal',
+                               'experimental', 'use_sell_signal')
+    check_conflicting_settings(config, 'ask_strategy', 'sell_profit_only',
+                               'experimental', 'sell_profit_only')
+    check_conflicting_settings(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
+                               'experimental', 'ignore_roi_if_buy_signal')
+
+    process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
+                               'experimental', 'use_sell_signal')
+    process_deprecated_setting(config, 'ask_strategy', 'sell_profit_only',
+                               'experimental', 'sell_profit_only')
+    process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
+                               'experimental', 'ignore_roi_if_buy_signal')

freqtrade/configuration/timerange.py

@@ -42,9 +42,10 @@ class TimeRange:
                   (r'^-(\d{10})$', (None, 'date')),
                   (r'^(\d{10})-$', ('date', None)),
                   (r'^(\d{10})-(\d{10})$', ('date', 'date')),
-                  (r'^(-\d+)$', (None, 'line')),
+                  (r'^-(\d{13})$', (None, 'date')),
-                  (r'^(\d+)-$', ('line', None)),
+                  (r'^(\d{13})-$', ('date', None)),
-                  (r'^(\d+)-(\d+)$', ('index', 'index'))]
+                  (r'^(\d{13})-(\d{13})$', ('date', 'date')),
+                  ]
         for rex, stype in syntax:
             # Apply the regular expression to text
             match = re.match(rex, text)
@@ -57,6 +58,8 @@ class TimeRange:
                     starts = rvals[index]
                     if stype[0] == 'date' and len(starts) == 8:
                         start = arrow.get(starts, 'YYYYMMDD').timestamp
+                    elif len(starts) == 13:
+                        start = int(starts) // 1000
                     else:
                         start = int(starts)
                     index += 1
@@ -64,6 +67,8 @@ class TimeRange:
                     stops = rvals[index]
                     if stype[1] == 'date' and len(stops) == 8:
                         stop = arrow.get(stops, 'YYYYMMDD').timestamp
+                    elif len(stops) == 13:
+                        stop = int(stops) // 1000
                     else:
                         stop = int(stops)
                 return TimeRange(stype[0], stype[1], start, stop)

freqtrade/constants.py

@@ -10,7 +10,7 @@ DEFAULT_TICKER_INTERVAL = 5  # min
 HYPEROPT_EPOCH = 100  # epochs
 RETRY_TIMEOUT = 30  # sec
 DEFAULT_STRATEGY = 'DefaultStrategy'
-DEFAULT_HYPEROPT = 'DefaultHyperOpts'
+DEFAULT_HYPEROPT = 'DefaultHyperOpt'
 DEFAULT_HYPEROPT_LOSS = 'DefaultHyperOptLoss'
 DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
 DEFAULT_DB_DRYRUN_URL = 'sqlite://'
@@ -114,7 +114,10 @@ CONF_SCHEMA = {
             'properties': {
                 'use_order_book': {'type': 'boolean'},
                 'order_book_min': {'type': 'number', 'minimum': 1},
-                'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50}
+                'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50},
+                'use_sell_signal': {'type': 'boolean'},
+                'sell_profit_only': {'type': 'boolean'},
+                'ignore_roi_if_buy_signal': {'type': 'boolean'}
             }
         },
         'order_types': {
@@ -144,7 +147,8 @@ CONF_SCHEMA = {
             'properties': {
                 'use_sell_signal': {'type': 'boolean'},
                 'sell_profit_only': {'type': 'boolean'},
-                'ignore_roi_if_buy_signal_true': {'type': 'boolean'}
+                'ignore_roi_if_buy_signal': {'type': 'boolean'},
+                'block_bad_exchanges': {'type': 'boolean'}
             }
         },
         'pairlist': {
@@ -262,6 +266,6 @@ CONF_SCHEMA = {
         'stake_amount',
         'dry_run',
         'bid_strategy',
-        'telegram'
+        'unfilledtimeout',
     ]
 }

freqtrade/data/btanalysis.py

@@ -93,7 +93,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
                             t.close_date.replace(tzinfo=pytz.UTC) if t.close_date else None,
                             t.calc_profit(), t.calc_profit_percent(),
                             t.open_rate, t.close_rate, t.amount,
-                            (t.close_date.timestamp() - t.open_date.timestamp()
+                            (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)
                                 if t.close_date else None),
                             t.sell_reason,
                             t.fee_open, t.fee_close,
@@ -150,15 +150,21 @@ def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "c
     return df_comb
 
 
-def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str) -> pd.DataFrame:
+def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
+                      timeframe: str) -> pd.DataFrame:
     """
     Adds a column `col_name` with the cumulative profit for the given trades array.
     :param df: DataFrame with date index
     :param trades: DataFrame containing trades (requires columns close_time and profitperc)
+    :param col_name: Column name that will be assigned the results
+    :param timeframe: Timeframe used during the operations
     :return: Returns df with one additional column, col_name, containing the cumulative profit.
     """
-    # Use groupby/sum().cumsum() to avoid errors when multiple trades sold at the same candle.
+    from freqtrade.exchange import timeframe_to_minutes
-    df[col_name] = trades.groupby('close_time')['profitperc'].sum().cumsum()
+    ticker_minutes = timeframe_to_minutes(timeframe)
+    # Resample to ticker_interval to make sure trades match candles
+    _trades_sum = trades.resample(f'{ticker_minutes}min', on='close_time')[['profitperc']].sum()
+    df.loc[:, col_name] = _trades_sum.cumsum()
     # Set first value to 0
     df.loc[df.iloc[0].name, col_name] = 0
     # FFill to get continuous

freqtrade/data/converter.py

@@ -114,3 +114,25 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
                       keys=['b_sum', 'b_size', 'bids', 'asks', 'a_size', 'a_sum'])
     # logger.info('order book %s', frame )
     return frame
+
+
+def trades_to_ohlcv(trades: list, timeframe: str) -> list:
+    """
+    Converts trades list to ohlcv list
+    :param trades: List of trades, as returned by ccxt.fetch_trades.
+    :param timeframe: Ticker timeframe to resample data to
+    :return: ohlcv timeframe as list (as returned by ccxt.fetch_ohlcv)
+    """
+    from freqtrade.exchange import timeframe_to_minutes
+    ticker_minutes = timeframe_to_minutes(timeframe)
+    df = pd.DataFrame(trades)
+    df['datetime'] = pd.to_datetime(df['datetime'])
+    df = df.set_index('datetime')
+
+    df_new = df['price'].resample(f'{ticker_minutes}min').ohlc()
+    df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum()
+    df_new['date'] = df_new.index.astype("int64") // 10 ** 6
+    # Drop 0 volume rows
+    df_new = df_new.dropna()
+    columns = ["date", "open", "high", "low", "close", "volume"]
+    return list(zip(*[df_new[x].values.tolist() for x in columns]))

freqtrade/data/dataprovider.py

@@ -6,7 +6,7 @@ Common Interface for bot and strategy to access data.
 """
 import logging
 from pathlib import Path
-from typing import List, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 from pandas import DataFrame
 
@@ -85,6 +85,14 @@ class DataProvider:
             logger.warning(f"No data found for ({pair}, {ticker_interval}).")
         return data
 
+    def market(self, pair: str) -> Optional[Dict[str, Any]]:
+        """
+        Return market data for the pair
+        :param pair: Pair to get the data for
+        :return: Market data dict from ccxt or None if market info is not available for the pair
+        """
+        return self._exchange.markets.get(pair)
+
     def ticker(self, pair: str):
         """
         Return last ticker data
@@ -92,9 +100,9 @@ class DataProvider:
         # TODO: Implement me
         pass
 
-    def orderbook(self, pair: str, maximum: int):
+    def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
         """
-        return latest orderbook data
+        fetch latest orderbook data
         :param pair: pair to get the data for
         :param maximum: Maximum number of orderbook entries to query
         :return: dict including bids/asks with a total of `maximum` entries.

freqtrade/data/history.py

@@ -17,7 +17,7 @@ from pandas import DataFrame
 
 from freqtrade import OperationalException, misc
 from freqtrade.configuration import TimeRange
-from freqtrade.data.converter import parse_ticker_dataframe
+from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
 from freqtrade.exchange import Exchange, timeframe_to_minutes
 
 logger = logging.getLogger(__name__)
@@ -33,20 +33,12 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
     start_index = 0
     stop_index = len(tickerlist)
 
-    if timerange.starttype == 'line':
+    if timerange.starttype == 'date':
-        stop_index = timerange.startts
-    if timerange.starttype == 'index':
-        start_index = timerange.startts
-    elif timerange.starttype == 'date':
         while (start_index < len(tickerlist) and
                tickerlist[start_index][0] < timerange.startts * 1000):
             start_index += 1
 
-    if timerange.stoptype == 'line':
+    if timerange.stoptype == 'date':
-        start_index = max(len(tickerlist) + timerange.stopts, 0)
-    if timerange.stoptype == 'index':
-        stop_index = timerange.stopts
-    elif timerange.stoptype == 'date':
         while (stop_index > 0 and
                tickerlist[stop_index-1][0] > timerange.stopts * 1000):
             stop_index -= 1
@@ -82,10 +74,42 @@ def store_tickerdata_file(datadir: Path, pair: str,
     misc.file_dump_json(filename, data, is_zip=is_zip)
 
 
+def load_trades_file(datadir: Path, pair: str,
+                     timerange: Optional[TimeRange] = None) -> List[Dict]:
+    """
+    Load a pair from file, either .json.gz or .json
+    :return: tradelist or empty list if unsuccesful
+    """
+    filename = pair_trades_filename(datadir, pair)
+    tradesdata = misc.file_load_json(filename)
+    if not tradesdata:
+        return []
+
+    return tradesdata
+
+
+def store_trades_file(datadir: Path, pair: str,
+                      data: list, is_zip: bool = True):
+    """
+    Stores tickerdata to file
+    """
+    filename = pair_trades_filename(datadir, pair)
+    misc.file_dump_json(filename, data, is_zip=is_zip)
+
+
+def _validate_pairdata(pair, pairdata, timerange: TimeRange):
+    if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000:
+        logger.warning('Missing data at start for pair %s, data starts at %s',
+                       pair, arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
+    if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000:
+        logger.warning('Missing data at end for pair %s, data ends at %s',
+                       pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
+
+
 def load_pair_history(pair: str,
                       ticker_interval: str,
                       datadir: Path,
-                      timerange: TimeRange = TimeRange(None, None, 0, 0),
+                      timerange: Optional[TimeRange] = None,
                       refresh_pairs: bool = False,
                       exchange: Optional[Exchange] = None,
                       fill_up_missing: bool = True,
@@ -116,13 +140,8 @@ def load_pair_history(pair: str,
     pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
 
     if pairdata:
-        if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000:
+        if timerange:
-            logger.warning('Missing data at start for pair %s, data starts at %s',
+            _validate_pairdata(pair, pairdata, timerange)
-                           pair, arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
-        if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000:
-            logger.warning('Missing data at end for pair %s, data ends at %s',
-                           pair,
-                           arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
         return parse_ticker_dataframe(pairdata, ticker_interval, pair=pair,
                                       fill_missing=fill_up_missing,
                                       drop_incomplete=drop_incomplete)
@@ -139,7 +158,7 @@ def load_data(datadir: Path,
               pairs: List[str],
               refresh_pairs: bool = False,
               exchange: Optional[Exchange] = None,
-              timerange: TimeRange = TimeRange(None, None, 0, 0),
+              timerange: Optional[TimeRange] = None,
               fill_up_missing: bool = True,
               ) -> Dict[str, DataFrame]:
     """
@@ -169,13 +188,20 @@ def pair_data_filename(datadir: Path, pair: str, ticker_interval: str) -> Path:
     return filename
 
 
-def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str,
+def pair_trades_filename(datadir: Path, pair: str) -> Path:
-                                  timerange: Optional[TimeRange]) -> Tuple[List[Any],
+    pair_s = pair.replace("/", "_")
-                                                                           Optional[int]]:
+    filename = datadir.joinpath(f'{pair_s}-trades.json.gz')
+    return filename
+
+
+def _load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str,
+                                   timerange: Optional[TimeRange]) -> Tuple[List[Any],
+                                                                            Optional[int]]:
     """
     Load cached data to download more data.
-    If timerange is passed in, checks wether data from an before the stored data will be downloaded.
+    If timerange is passed in, checks whether data from an before the stored data will be
-    If that's the case than what's available should be completely overwritten.
+    downloaded.
+    If that's the case then what's available should be completely overwritten.
     Only used by download_pair_history().
     """
 
@@ -238,7 +264,7 @@ def download_pair_history(datadir: Path,
             f'and store in {datadir}.'
         )
 
-        data, since_ms = load_cached_data_for_updating(datadir, pair, ticker_interval, timerange)
+        data, since_ms = _load_cached_data_for_updating(datadir, pair, ticker_interval, timerange)
 
         logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None')
         logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
@@ -266,7 +292,7 @@ def download_pair_history(datadir: Path,
 
 
 def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
-                                dl_path: Path, timerange: TimeRange,
+                                dl_path: Path, timerange: Optional[TimeRange] = None,
                                 erase=False) -> List[str]:
     """
     Refresh stored ohlcv data for backtesting and hyperopt operations.
@@ -294,6 +320,92 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
     return pairs_not_available
 
 
+def download_trades_history(datadir: Path,
+                            exchange: Exchange,
+                            pair: str,
+                            timerange: Optional[TimeRange] = None) -> bool:
+    """
+    Download trade history from the exchange.
+    Appends to previously downloaded trades data.
+    """
+    try:
+
+        since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None
+
+        trades = load_trades_file(datadir, pair)
+
+        from_id = trades[-1]['id'] if trades else None
+
+        logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
+        logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
+
+        new_trades = exchange.get_historic_trades(pair=pair,
+                                                  since=since if since else
+                                                  int(arrow.utcnow().shift(
+                                                      days=-30).float_timestamp) * 1000,
+                                                  #  until=xxx,
+                                                  from_id=from_id,
+                                                  )
+        trades.extend(new_trades[1])
+        store_trades_file(datadir, pair, trades)
+
+        logger.debug("New Start: %s", trades[0]['datetime'])
+        logger.debug("New End: %s", trades[-1]['datetime'])
+        logger.info(f"New Amount of trades: {len(trades)}")
+        return True
+
+    except Exception as e:
+        logger.error(
+            f'Failed to download historic trades for pair: "{pair}". '
+            f'Error: {e}'
+        )
+        return False
+
+
+def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
+                                 timerange: TimeRange, erase=False) -> List[str]:
+    """
+    Refresh stored trades data.
+    Used by freqtrade download-data
+    :return: Pairs not available
+    """
+    pairs_not_available = []
+    for pair in pairs:
+        if pair not in exchange.markets:
+            pairs_not_available.append(pair)
+            logger.info(f"Skipping pair {pair}...")
+            continue
+
+        dl_file = pair_trades_filename(datadir, pair)
+        if erase and dl_file.exists():
+            logger.info(
+                f'Deleting existing data for pair {pair}.')
+            dl_file.unlink()
+
+        logger.info(f'Downloading trades for pair {pair}.')
+        download_trades_history(datadir=datadir, exchange=exchange,
+                                pair=pair,
+                                timerange=timerange)
+    return pairs_not_available
+
+
+def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
+                            datadir: Path, timerange: TimeRange, erase=False) -> None:
+    """
+    Convert stored trades data to ohlcv data
+    """
+    for pair in pairs:
+        trades = load_trades_file(datadir, pair)
+        for timeframe in timeframes:
+            ohlcv_file = pair_data_filename(datadir, pair, timeframe)
+            if erase and ohlcv_file.exists():
+                logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.')
+                ohlcv_file.unlink()
+            ohlcv = trades_to_ohlcv(trades, timeframe)
+            # Store ohlcv
+            store_tickerdata_file(datadir, pair, timeframe, data=ohlcv)
+
+
 def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
     """
     Get the maximum timeframe for the given backtest data

freqtrade/edge/__init__.py

@@ -77,8 +77,10 @@ class Edge:
 
         self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift(
             days=-1 * self._since_number_of_days).format('YYYYMMDD'))
-
+        if config.get('fee'):
-        self.fee = self.exchange.get_fee()
+            self.fee = config['fee']
+        else:
+            self.fee = self.exchange.get_fee()
 
     def calculate(self) -> bool:
         pairs = self.config['exchange']['pair_whitelist']

freqtrade/exchange/__init__.py

@@ -1,13 +1,16 @@
-from freqtrade.exchange.exchange import Exchange  # noqa: F401
+from freqtrade.exchange.exchange import Exchange, MAP_EXCHANGE_CHILDCLASS  # noqa: F401
 from freqtrade.exchange.exchange import (get_exchange_bad_reason,  # noqa: F401
                                          is_exchange_bad,
-                                         is_exchange_available,
+                                         is_exchange_known_ccxt,
                                          is_exchange_officially_supported,
+                                         ccxt_exchanges,
                                          available_exchanges)
 from freqtrade.exchange.exchange import (timeframe_to_seconds,  # noqa: F401
                                          timeframe_to_minutes,
                                          timeframe_to_msecs,
                                          timeframe_to_next_date,
                                          timeframe_to_prev_date)
+from freqtrade.exchange.exchange import (market_is_active,  # noqa: F401
+                                         symbol_is_pair)
 from freqtrade.exchange.kraken import Kraken  # noqa: F401
 from freqtrade.exchange.binance import Binance  # noqa: F401

freqtrade/exchange/binance.py

@@ -16,6 +16,8 @@ class Binance(Exchange):
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
         "order_time_in_force": ['gtc', 'fok', 'ioc'],
+        "trades_pagination": "id",
+        "trades_pagination_arg": "fromId",
     }
 
     def get_order_book(self, pair: str, limit: int = 100) -> dict:

freqtrade/exchange/exchange.py

@@ -22,16 +22,92 @@ from freqtrade import (DependencyException, InvalidOrderException,
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.misc import deep_merge_dicts
 
+
 logger = logging.getLogger(__name__)
 
 
 API_RETRY_COUNT = 4
 BAD_EXCHANGES = {
-    "bitmex": "Various reasons",
+    "bitmex": "Various reasons.",
     "bitstamp": "Does not provide history. "
                 "Details in https://github.com/freqtrade/freqtrade/issues/1983",
+    "hitbtc": "This API cannot be used with Freqtrade. "
+              "Use `hitbtc2` exchange id to access this exchange.",
+    **dict.fromkeys([
+        'adara',
+        'anxpro',
+        'bigone',
+        'coinbase',
+        'coinexchange',
+        'coinmarketcap',
+        'lykke',
+        'xbtce',
+        ], "Does not provide timeframes. ccxt fetchOHLCV: False"),
+    **dict.fromkeys([
+        'bcex',
+        'bit2c',
+        'bitbay',
+        'bitflyer',
+        'bitforex',
+        'bithumb',
+        'bitso',
+        'bitstamp1',
+        'bl3p',
+        'braziliex',
+        'btcbox',
+        'btcchina',
+        'btctradeim',
+        'btctradeua',
+        'bxinth',
+        'chilebit',
+        'coincheck',
+        'coinegg',
+        'coinfalcon',
+        'coinfloor',
+        'coingi',
+        'coinmate',
+        'coinone',
+        'coinspot',
+        'coolcoin',
+        'crypton',
+        'deribit',
+        'exmo',
+        'exx',
+        'flowbtc',
+        'foxbit',
+        'fybse',
+        # 'hitbtc',
+        'ice3x',
+        'independentreserve',
+        'indodax',
+        'itbit',
+        'lakebtc',
+        'latoken',
+        'liquid',
+        'livecoin',
+        'luno',
+        'mixcoins',
+        'negociecoins',
+        'nova',
+        'paymium',
+        'southxchange',
+        'stronghold',
+        'surbitcoin',
+        'therock',
+        'tidex',
+        'vaultoro',
+        'vbtc',
+        'virwox',
+        'yobit',
+        'zaif',
+        ], "Does not provide timeframes. ccxt fetchOHLCV: emulated"),
     }
 
+MAP_EXCHANGE_CHILDCLASS = {
+    'binanceus': 'binance',
+    'binanceje': 'binance',
+}
+
 
 def retrier_async(f):
     async def wrapper(*args, **kwargs):
@@ -72,6 +148,8 @@ def retrier(f):
 class Exchange:
 
     _config: Dict = {}
+
+    # Parameters to add directly to buy/sell calls (like agreeing to trading agreement)
     _params: Dict = {}
 
     # Dict to specify which options each exchange implements
@@ -82,10 +160,13 @@ class Exchange:
         "order_time_in_force": ["gtc"],
         "ohlcv_candle_limit": 500,
         "ohlcv_partial_candle": True,
+        "trades_pagination": "time",  # Possible are "time" or "id"
+        "trades_pagination_arg": "since",
+
     }
     _ft_has: Dict = {}
 
-    def __init__(self, config: dict) -> None:
+    def __init__(self, config: dict, validate: bool = True) -> None:
         """
         Initializes this module with the given config,
         it does basic validation whether the specified exchange and pairs are valid.
@@ -125,6 +206,9 @@ class Exchange:
         self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit']
         self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle']
 
+        self._trades_pagination = self._ft_has['trades_pagination']
+        self._trades_pagination_arg = self._ft_has['trades_pagination_arg']
+
         # Initialize ccxt objects
         self._api = self._init_ccxt(
             exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config'))
@@ -133,20 +217,21 @@ class Exchange:
 
         logger.info('Using Exchange "%s"', self.name)
 
+        if validate:
+            # Check if timeframe is available
+            self.validate_timeframes(config.get('ticker_interval'))
+
+            # Initial markets load
+            self._load_markets()
+
+            # Check if all pairs are available
+            self.validate_pairs(config['exchange']['pair_whitelist'])
+            self.validate_ordertypes(config.get('order_types', {}))
+            self.validate_order_time_in_force(config.get('order_time_in_force', {}))
+
         # Converts the interval provided in minutes in config to seconds
         self.markets_refresh_interval: int = exchange_config.get(
             "markets_refresh_interval", 60) * 60
-        # Initial markets load
-        self._load_markets()
-
-        # Check if all pairs are available
-        self.validate_pairs(config['exchange']['pair_whitelist'])
-        self.validate_ordertypes(config.get('order_types', {}))
-        self.validate_order_time_in_force(config.get('order_time_in_force', {}))
-
-        if config.get('ticker_interval'):
-            # Check if timeframe is available
-            self.validate_timeframes(config['ticker_interval'])
 
     def __del__(self):
         """
@@ -165,7 +250,7 @@ class Exchange:
         # Find matching class for the given exchange name
         name = exchange_config['name']
 
-        if not is_exchange_available(name, ccxt_module):
+        if not is_exchange_known_ccxt(name, ccxt_module):
             raise OperationalException(f'Exchange {name} is not supported by ccxt')
 
         ex_config = {
@@ -199,6 +284,10 @@ class Exchange:
         """exchange ccxt id"""
         return self._api.id
 
+    @property
+    def timeframes(self) -> List[str]:
+        return list((self._api.timeframes or {}).keys())
+
     @property
     def markets(self) -> Dict:
         """exchange ccxt markets"""
@@ -207,6 +296,28 @@ class Exchange:
             self._load_markets()
         return self._api.markets
 
+    def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None,
+                    pairs_only: bool = False, active_only: bool = False) -> Dict:
+        """
+        Return exchange ccxt markets, filtered out by base currency and quote currency
+        if this was requested in parameters.
+
+        TODO: consider moving it to the Dataprovider
+        """
+        markets = self.markets
+        if not markets:
+            raise OperationalException("Markets were not loaded.")
+
+        if base_currencies:
+            markets = {k: v for k, v in markets.items() if v['base'] in base_currencies}
+        if quote_currencies:
+            markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies}
+        if pairs_only:
+            markets = {k: v for k, v in markets.items() if symbol_is_pair(v['symbol'])}
+        if active_only:
+            markets = {k: v for k, v in markets.items() if market_is_active(v)}
+        return markets
+
     def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame:
         if pair_interval in self._klines:
             return self._klines[pair_interval].copy() if copy else self._klines[pair_interval]
@@ -291,7 +402,7 @@ class Exchange:
                 return pair
         raise DependencyException(f"Could not combine {curr_1} and {curr_2} to get a valid pair.")
 
-    def validate_timeframes(self, timeframe: List[str]) -> None:
+    def validate_timeframes(self, timeframe: Optional[str]) -> None:
         """
         Checks if ticker interval from config is a supported timeframe on the exchange
         """
@@ -304,10 +415,9 @@ class Exchange:
                 f"for the exchange \"{self.name}\" and this exchange "
                 f"is therefore not supported. ccxt fetchOHLCV: {self.exchange_has('fetchOHLCV')}")
 
-        timeframes = self._api.timeframes
+        if timeframe and (timeframe not in self.timeframes):
-        if timeframe not in timeframes:
             raise OperationalException(
-                f'Invalid ticker {timeframe}, this Exchange supports {timeframes}')
+                f"Invalid ticker interval '{timeframe}'. This exchange supports: {self.timeframes}")
 
     def validate_ordertypes(self, order_types: Dict) -> None:
         """
@@ -665,6 +775,154 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(f'Could not fetch ticker data. Msg: {e}') from e
 
+    @retrier_async
+    async def _async_fetch_trades(self, pair: str,
+                                  since: Optional[int] = None,
+                                  params: Optional[dict] = None) -> List[Dict]:
+        """
+        Asyncronously gets trade history using fetch_trades.
+        Handles exchange errors, does one call to the exchange.
+        :param pair: Pair to fetch trade data for
+        :param since: Since as integer timestamp in milliseconds
+        returns: List of dicts containing trades
+        """
+        try:
+            # fetch trades asynchronously
+            if params:
+                logger.debug("Fetching trades for pair %s, params: %s ", pair, params)
+                trades = await self._api_async.fetch_trades(pair, params=params, limit=1000)
+            else:
+                logger.debug(
+                    "Fetching trades for pair %s, since %s %s...",
+                    pair,  since,
+                    '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else ''
+                )
+                trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
+            return trades
+        except ccxt.NotSupported as e:
+            raise OperationalException(
+                f'Exchange {self._api.name} does not support fetching historical trade data.'
+                f'Message: {e}') from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(f'Could not load trade history due to {e.__class__.__name__}. '
+                                 f'Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e
+
+    async def _async_get_trade_history_id(self, pair: str,
+                                          until: int,
+                                          since: Optional[int] = None,
+                                          from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
+        """
+        Asyncronously gets trade history using fetch_trades
+        use this when exchange uses id-based iteration (check `self._trades_pagination`)
+        :param pair: Pair to fetch trade data for
+        :param since: Since as integer timestamp in milliseconds
+        :param until: Until as integer timestamp in milliseconds
+        :param from_id: Download data starting with ID (if id is known). Ignores "since" if set.
+        returns tuple: (pair, trades-list)
+        """
+
+        trades: List[Dict] = []
+
+        if not from_id:
+            # Fetch first elements using timebased method to get an ID to paginate on
+            # Depending on the Exchange, this can introduce a drift at the start of the interval
+            # of up to an hour.
+            # e.g. Binance returns the "last 1000" candles within a 1h time interval
+            # - so we will miss the first trades.
+            t = await self._async_fetch_trades(pair, since=since)
+            from_id = t[-1]['id']
+            trades.extend(t[:-1])
+        while True:
+            t = await self._async_fetch_trades(pair,
+                                               params={self._trades_pagination_arg: from_id})
+            if len(t):
+                # Skip last id since its the key for the next call
+                trades.extend(t[:-1])
+                if from_id == t[-1]['id'] or t[-1]['timestamp'] > until:
+                    logger.debug(f"Stopping because from_id did not change. "
+                                 f"Reached {t[-1]['timestamp']} > {until}")
+                    # Reached the end of the defined-download period - add last trade as well.
+                    trades.extend(t[-1:])
+                    break
+
+                from_id = t[-1]['id']
+            else:
+                break
+
+        return (pair, trades)
+
+    async def _async_get_trade_history_time(self, pair: str, until: int,
+                                            since: Optional[int] = None) -> Tuple[str, List]:
+        """
+        Asyncronously gets trade history using fetch_trades,
+        when the exchange uses time-based iteration (check `self._trades_pagination`)
+        :param pair: Pair to fetch trade data for
+        :param since: Since as integer timestamp in milliseconds
+        :param until: Until as integer timestamp in milliseconds
+        returns tuple: (pair, trades-list)
+        """
+
+        trades: List[Dict] = []
+        while True:
+            t = await self._async_fetch_trades(pair, since=since)
+            if len(t):
+                since = t[-1]['timestamp']
+                trades.extend(t)
+                # Reached the end of the defined-download period
+                if until and t[-1]['timestamp'] > until:
+                    logger.debug(
+                        f"Stopping because until was reached. {t[-1]['timestamp']} > {until}")
+                    break
+            else:
+                break
+
+        return (pair, trades)
+
+    async def _async_get_trade_history(self, pair: str,
+                                       since: Optional[int] = None,
+                                       until: Optional[int] = None,
+                                       from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
+        """
+        Async wrapper handling downloading trades using either time or id based methods.
+        """
+
+        if self._trades_pagination == 'time':
+            return await self._async_get_trade_history_time(
+                pair=pair, since=since,
+                until=until or ccxt.Exchange.milliseconds())
+        elif self._trades_pagination == 'id':
+            return await self._async_get_trade_history_id(
+                pair=pair, since=since,
+                until=until or ccxt.Exchange.milliseconds(), from_id=from_id
+            )
+        else:
+            raise OperationalException(f"Exchange {self.name} does use neither time, "
+                                       f"nor id based pagination")
+
+    def get_historic_trades(self, pair: str,
+                            since: Optional[int] = None,
+                            until: Optional[int] = None,
+                            from_id: Optional[str] = None) -> Tuple[str, List]:
+        """
+        Gets candle history using asyncio and returns the list of candles.
+        Handles all async doing.
+        Async over one pair, assuming we get `_ohlcv_candle_limit` candles per call.
+        :param pair: Pair to download
+        :param ticker_interval: Interval to get
+        :param since: Timestamp in milliseconds to get history from
+        :param until: Timestamp in milliseconds. Defaults to current timestamp if not defined.
+        :param from_id: Download data starting with ID (if id is known)
+        :returns List of tickers
+        """
+        if not self.exchange_has("fetchTrades"):
+            raise OperationalException("This exchange does not suport downloading Trades.")
+
+        return asyncio.get_event_loop().run_until_complete(
+            self._async_get_trade_history(pair=pair, since=since,
+                                          until=until, from_id=from_id))
+
     @retrier
     def cancel_order(self, order_id: str, pair: str) -> None:
         if self._config['dry_run']:
@@ -768,18 +1026,29 @@ def get_exchange_bad_reason(exchange_name: str) -> str:
     return BAD_EXCHANGES.get(exchange_name, "")
 
 
-def is_exchange_available(exchange_name: str, ccxt_module=None) -> bool:
+def is_exchange_known_ccxt(exchange_name: str, ccxt_module=None) -> bool:
-    return exchange_name in available_exchanges(ccxt_module)
+    return exchange_name in ccxt_exchanges(ccxt_module)
 
 
 def is_exchange_officially_supported(exchange_name: str) -> bool:
     return exchange_name in ['bittrex', 'binance']
 
 
-def available_exchanges(ccxt_module=None) -> List[str]:
+def ccxt_exchanges(ccxt_module=None) -> List[str]:
+    """
+    Return the list of all exchanges known to ccxt
+    """
     return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges
 
 
+def available_exchanges(ccxt_module=None) -> List[str]:
+    """
+    Return exchanges available to the bot, i.e. non-bad exchanges in the ccxt list
+    """
+    exchanges = ccxt_exchanges(ccxt_module)
+    return [x for x in exchanges if not is_exchange_bad(x)]
+
+
 def timeframe_to_seconds(ticker_interval: str) -> int:
     """
     Translates the timeframe interval value written in the human readable
@@ -830,3 +1099,27 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime:
     new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000,
                                                   ROUND_UP) // 1000
     return datetime.fromtimestamp(new_timestamp, tz=timezone.utc)
+
+
+def symbol_is_pair(market_symbol: str, base_currency: str = None, quote_currency: str = None):
+    """
+    Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the
+    quote currency separated by '/' character. If base_currency and/or quote_currency is passed,
+    it also checks that the symbol contains appropriate base and/or quote currency part before
+    and after the separating character correspondingly.
+    """
+    symbol_parts = market_symbol.split('/')
+    return (len(symbol_parts) == 2 and
+            (symbol_parts[0] == base_currency if base_currency else len(symbol_parts[0]) > 0) and
+            (symbol_parts[1] == quote_currency if quote_currency else len(symbol_parts[1]) > 0))
+
+
+def market_is_active(market):
+    """
+    Return True if the market is active.
+    """
+    # "It's active, if the active flag isn't explicitly set to false. If it's missing or
+    # true then it's true. If it's undefined, then it's most likely true, but not 100% )"
+    # See https://github.com/ccxt/ccxt/issues/4874,
+    # https://github.com/ccxt/ccxt/issues/4075#issuecomment-434760520
+    return market.get('active', True) is not False

freqtrade/exchange/kraken.py

@@ -14,6 +14,10 @@ logger = logging.getLogger(__name__)
 class Kraken(Exchange):
 
     _params: Dict = {"trading_agreement": "agree"}
+    _ft_has: Dict = {
+        "trades_pagination": "id",
+        "trades_pagination_arg": "since",
+    }
 
     @retrier
     def get_balances(self) -> dict:

freqtrade/freqtradebot.py

@@ -1,32 +1,32 @@
 """
 Freqtrade is the main module of this bot. It contains the class Freqtrade()
 """
-
 import copy
 import logging
 import traceback
 from datetime import datetime
 from math import isclose
+from os import getpid
 from typing import Any, Dict, List, Optional, Tuple
 
 import arrow
 from requests.exceptions import RequestException
 
-from freqtrade import (DependencyException, OperationalException, InvalidOrderException,
+from freqtrade import (DependencyException, InvalidOrderException, __version__,
-                       __version__, constants, persistence)
+                       constants, persistence)
+from freqtrade.configuration import validate_config_consistency
 from freqtrade.data.converter import order_book_to_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
-from freqtrade.configuration import validate_config_consistency
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
 from freqtrade.persistence import Trade
+from freqtrade.resolvers import (ExchangeResolver, PairListResolver,
+                                 StrategyResolver)
 from freqtrade.rpc import RPCManager, RPCMessageType
-from freqtrade.resolvers import ExchangeResolver, StrategyResolver, PairListResolver
 from freqtrade.state import State
-from freqtrade.strategy.interface import SellType, IStrategy
+from freqtrade.strategy.interface import IStrategy, SellType
 from freqtrade.wallets import Wallets
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -51,13 +51,15 @@ class FreqtradeBot:
         # Init objects
         self.config = config
 
+        self._heartbeat_msg = 0
+
+        self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60)
+
         self.strategy: IStrategy = StrategyResolver(self.config).strategy
 
         # Check config consistency here since strategies can set certain options
         validate_config_consistency(config)
 
-        self.rpc: RPCManager = RPCManager(self)
-
         self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange
 
         self.wallets = Wallets(self.config, self.exchange)
@@ -84,6 +86,13 @@ class FreqtradeBot:
         initial_state = self.config.get('initial_state')
         self.state = State[initial_state.upper()] if initial_state else State.STOPPED
 
+        # RPC runs in separate threads, can start handling external commands just after
+        # initialization, even before Freqtradebot has a chance to start its throttling,
+        # so anything in the Freqtradebot instance should be ready (initialized), including
+        # the initial state of the bot.
+        # Keep this at the end of this initialization method.
+        self.rpc: RPCManager = RPCManager(self)
+
     def cleanup(self) -> None:
         """
         Cleanup pending resources on an already stopped bot
@@ -135,18 +144,22 @@ class FreqtradeBot:
                                   self.strategy.informative_pairs())
 
         # First process current opened trades
-        for trade in trades:
+        self.process_maybe_execute_sells(trades)
-            self.process_maybe_execute_sell(trade)
 
         # Then looking for buy opportunities
         if len(trades) < self.config['max_open_trades']:
-            self.process_maybe_execute_buy()
+            self.process_maybe_execute_buys()
 
         if 'unfilledtimeout' in self.config:
             # Check and handle any timed out open orders
             self.check_handle_timedout()
             Trade.session.flush()
 
+        if (self.heartbeat_interval
+           and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)):
+            logger.info(f"Bot heartbeat. PID={getpid()}")
+            self._heartbeat_msg = arrow.utcnow().timestamp
+
     def _extend_whitelist_with_trades(self, whitelist: List[str], trades: List[Any]):
         """
         Extend whitelist with pairs from open trades
@@ -262,11 +275,10 @@ class FreqtradeBot:
         Checks pairs as long as the open trade count is below `max_open_trades`.
         :return: True if at least one trade has been created.
         """
-        interval = self.strategy.ticker_interval
         whitelist = copy.deepcopy(self.active_pair_whitelist)
 
         if not whitelist:
-            logger.warning("Whitelist is empty.")
+            logger.info("Active pair whitelist is empty.")
             return False
 
         # Remove currently opened and latest pairs from whitelist
@@ -276,7 +288,8 @@ class FreqtradeBot:
                 logger.debug('Ignoring %s in pair whitelist', trade.pair)
 
         if not whitelist:
-            logger.info("No currency pair in whitelist, but checking to sell open trades.")
+            logger.info("No currency pair in active pair whitelist, "
+                        "but checking to sell open trades.")
             return False
 
         buycount = 0
@@ -285,8 +298,10 @@ class FreqtradeBot:
             if self.strategy.is_pair_locked(_pair):
                 logger.info(f"Pair {_pair} is currently locked.")
                 continue
+
             (buy, sell) = self.strategy.get_signal(
-                _pair, interval, self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval))
+                _pair, self.strategy.ticker_interval,
+                self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval))
 
             if buy and not sell and len(Trade.get_open_trades()) < self.config['max_open_trades']:
                 stake_amount = self._get_trade_stake_amount(_pair)
@@ -431,52 +446,48 @@ class FreqtradeBot:
 
         return True
 
-    def process_maybe_execute_buy(self) -> None:
+    def process_maybe_execute_buys(self) -> None:
         """
-        Tries to execute a buy trade in a safe way
+        Tries to execute buy orders for trades in a safe way
-        :return: True if executed
         """
         try:
             # Create entity and execute trade
             if not self.create_trades():
-                logger.info('Found no buy signals for whitelisted currencies. Trying again...')
+                logger.debug('Found no buy signals for whitelisted currencies. Trying again...')
         except DependencyException as exception:
             logger.warning('Unable to create trade: %s', exception)
 
-    def process_maybe_execute_sell(self, trade: Trade) -> bool:
+    def process_maybe_execute_sells(self, trades: List[Any]) -> None:
         """
-        Tries to execute a sell trade
+        Tries to execute sell orders for trades in a safe way
-        :return: True if executed
         """
-        try:
+        result = False
-            self.update_trade_state(trade)
+        for trade in trades:
+            try:
+                self.update_trade_state(trade)
 
-            if self.strategy.order_types.get('stoploss_on_exchange') and trade.is_open:
+                if (self.strategy.order_types.get('stoploss_on_exchange') and
-                result = self.handle_stoploss_on_exchange(trade)
+                        self.handle_stoploss_on_exchange(trade)):
-                if result:
+                    result = True
-                    self.wallets.update()
+                    continue
-                    return result
-
-            if trade.is_open and trade.open_order_id is None:
                 # Check if we can sell our current pair
-                result = self.handle_trade(trade)
+                if trade.open_order_id is None and self.handle_trade(trade):
+                    result = True
 
-                # Updating wallets if any trade occured
+            except DependencyException as exception:
-                if result:
+                logger.warning('Unable to sell trade: %s', exception)
-                    self.wallets.update()
 
-                return result
+        # Updating wallets if any trade occured
+        if result:
+            self.wallets.update()
 
-        except DependencyException as exception:
+    def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float:
-            logger.warning('Unable to sell trade: %s', exception)
-        return False
-
-    def get_real_amount(self, trade: Trade, order: Dict) -> float:
         """
         Get real amount for the trade
         Necessary for exchanges which charge fees in base currency (e.g. binance)
         """
-        order_amount = order['amount']
+        if order_amount is None:
+            order_amount = order['amount']
         # Only run for closed orders
         if trade.fee_open == 0 or order['status'] == 'open':
             return order_amount
@@ -513,7 +524,7 @@ class FreqtradeBot:
 
         if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC):
             logger.warning(f"Amount {amount} does not match amount {trade.amount}")
-            raise OperationalException("Half bought? Amounts don't match")
+            raise DependencyException("Half bought? Amounts don't match")
         real_amount = amount - fee_abs
         if fee_abs != 0:
             logger.info(f"Applying fee on amount for {trade} "
@@ -541,7 +552,7 @@ class FreqtradeBot:
                     # Fee was applied, so set to 0
                     trade.fee_open = 0
 
-            except OperationalException as exception:
+            except DependencyException as exception:
                 logger.warning("Could not update trade amount: %s", exception)
 
             trade.update(order)
@@ -575,18 +586,20 @@ class FreqtradeBot:
         :return: True if trade has been sold, False otherwise
         """
         if not trade.is_open:
-            raise ValueError(f'Attempt to handle closed trade: {trade}')
+            raise DependencyException(f'Attempt to handle closed trade: {trade}')
 
         logger.debug('Handling %s ...', trade)
 
         (buy, sell) = (False, False)
-        experimental = self.config.get('experimental', {})
+
-        if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
+        config_ask_strategy = self.config.get('ask_strategy', {})
+
+        if (config_ask_strategy.get('use_sell_signal', True) or
+                config_ask_strategy.get('ignore_roi_if_buy_signal')):
             (buy, sell) = self.strategy.get_signal(
                 trade.pair, self.strategy.ticker_interval,
                 self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval))
 
-        config_ask_strategy = self.config.get('ask_strategy', {})
         if config_ask_strategy.get('use_order_book', False):
             logger.info('Using order book for selling...')
             # logger.debug('Order book %s',orderBook)
@@ -708,7 +721,7 @@ class FreqtradeBot:
         if trade.stop_loss > float(order['info']['stopPrice']):
             # we check if the update is neccesary
             update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60)
-            if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() > update_beat:
+            if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat:
                 # cancelling the current stoploss on exchange first
                 logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s})'
                             'in order to add another one ...', order['id'])
@@ -750,8 +763,8 @@ class FreqtradeBot:
         """
         buy_timeout = self.config['unfilledtimeout']['buy']
         sell_timeout = self.config['unfilledtimeout']['sell']
-        buy_timeoutthreashold = arrow.utcnow().shift(minutes=-buy_timeout).datetime
+        buy_timeout_threshold = arrow.utcnow().shift(minutes=-buy_timeout).datetime
-        sell_timeoutthreashold = arrow.utcnow().shift(minutes=-sell_timeout).datetime
+        sell_timeout_threshold = arrow.utcnow().shift(minutes=-sell_timeout).datetime
 
         for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all():
             try:
@@ -775,21 +788,18 @@ class FreqtradeBot:
                 self.wallets.update()
                 continue
 
-            # Handle cancelled on exchange
+            if ((order['side'] == 'buy' and order['status'] == 'canceled')
-            if order['status'] == 'canceled':
+                or (order['status'] == 'open'
-                if order['side'] == 'buy':
+                    and order['side'] == 'buy' and ordertime < buy_timeout_threshold)):
-                    self.handle_buy_order_full_cancel(trade, "canceled on Exchange")
+
-                elif order['side'] == 'sell':
+                self.handle_timedout_limit_buy(trade, order)
-                    self.handle_timedout_limit_sell(trade, order)
+                self.wallets.update()
-                    self.wallets.update()
+
-            # Check if order is still actually open
+            elif ((order['side'] == 'sell' and order['status'] == 'canceled')
-            elif order['status'] == 'open':
+                  or (order['status'] == 'open'
-                if order['side'] == 'buy' and ordertime < buy_timeoutthreashold:
+                      and order['side'] == 'sell' and ordertime < sell_timeout_threshold)):
-                    self.handle_timedout_limit_buy(trade, order)
+                self.handle_timedout_limit_sell(trade, order)
-                    self.wallets.update()
+                self.wallets.update()
-                elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold:
-                    self.handle_timedout_limit_sell(trade, order)
-                    self.wallets.update()
 
     def handle_buy_order_full_cancel(self, trade: Trade, reason: str) -> None:
         """Close trade in database and send message"""
@@ -805,16 +815,33 @@ class FreqtradeBot:
         """Buy timeout - cancel order
         :return: True if order was fully cancelled
         """
-        self.exchange.cancel_order(trade.open_order_id, trade.pair)
+        reason = "cancelled due to timeout"
-        if order['remaining'] == order['amount']:
+        if order['status'] != 'canceled':
+            corder = self.exchange.cancel_order(trade.open_order_id, trade.pair)
+        else:
+            # Order was cancelled already, so we can reuse the existing dict
+            corder = order
+            reason = "canceled on Exchange"
+
+        if corder['remaining'] == corder['amount']:
             # if trade is not partially completed, just delete the trade
-            self.handle_buy_order_full_cancel(trade, "cancelled due to timeout")
+            self.handle_buy_order_full_cancel(trade, reason)
             return True
 
         # if trade is partially complete, edit the stake details for the trade
         # and close the order
-        trade.amount = order['amount'] - order['remaining']
+        trade.amount = corder['amount'] - corder['remaining']
         trade.stake_amount = trade.amount * trade.open_rate
+        # verify if fees were taken from amount to avoid problems during selling
+        try:
+            new_amount = self.get_real_amount(trade, corder, trade.amount)
+            if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
+                trade.amount = new_amount
+                # Fee was applied, so set to 0
+                trade.fee_open = 0
+        except DependencyException as e:
+            logger.warning("Could not update trade amount: %s", e)
+
         trade.open_order_id = None
         logger.info('Partial buy order timeout for %s.', trade)
         self.rpc.send_msg({

freqtrade/indicator_helpers.py

@@ -1,40 +0,0 @@
-from math import cos, exp, pi, sqrt
-
-import numpy as np
-import talib as ta
-from pandas import Series
-
-
-def went_up(series: Series) -> bool:
-    return series > series.shift(1)
-
-
-def went_down(series: Series) -> bool:
-    return series < series.shift(1)
-
-
-def ehlers_super_smoother(series: Series, smoothing: float = 6) -> Series:
-    magic = pi * sqrt(2) / smoothing
-    a1 = exp(-magic)
-    coeff2 = 2 * a1 * cos(magic)
-    coeff3 = -a1 * a1
-    coeff1 = (1 - coeff2 - coeff3) / 2
-
-    filtered = series.copy()
-
-    for i in range(2, len(series)):
-        filtered.iloc[i] = coeff1 * (series.iloc[i] + series.iloc[i-1]) + \
-            coeff2 * filtered.iloc[i-1] + coeff3 * filtered.iloc[i-2]
-
-    return filtered
-
-
-def fishers_inverse(series: Series, smoothing: float = 0) -> np.ndarray:
-    """ Does a smoothed fishers inverse transformation.
-        Can be used with any oscillator that goes from 0 to 100 like RSI or MFI """
-    v1 = 0.1 * (series - 50)
-    if smoothing > 0:
-        v2 = ta.WMA(v1.values, timeperiod=smoothing)
-    else:
-        v2 = v1
-    return (np.exp(2 * v2)-1) / (np.exp(2 * v2) + 1)

freqtrade/misc.py

@@ -72,8 +72,10 @@ def json_load(datafile: IO):
 
 def file_load_json(file):
 
-    gzipfile = file.with_suffix(file.suffix + '.gz')
+    if file.suffix != ".gz":
-
+        gzipfile = file.with_suffix(file.suffix + '.gz')
+    else:
+        gzipfile = file
     # Try gzip file first, otherwise regular json file.
     if gzipfile.is_file():
         logger.debug('Loading ticker data from file %s', gzipfile)
@@ -121,3 +123,7 @@ def round_dict(d, n):
     Rounds float values in the dict to n digits after the decimal point.
     """
     return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()}
+
+
+def plural(num, singular: str, plural: str = None) -> str:
+    return singular if (num == 1 or num == -1) else plural or singular + 's'

freqtrade/optimize/backtesting.py

@@ -63,9 +63,12 @@ class Backtesting:
         self.config['exchange']['uid'] = ''
         self.config['dry_run'] = True
         self.strategylist: List[IStrategy] = []
-
         self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange
-        self.fee = self.exchange.get_fee()
+
+        if config.get('fee'):
+            self.fee = config['fee']
+        else:
+            self.fee = self.exchange.get_fee()
 
         if self.config.get('runmode') != RunMode.HYPEROPT:
             self.dataprovider = DataProvider(self.config, self.exchange)
@@ -146,8 +149,8 @@ class Backtesting:
             len(results[results.profit_abs < 0])
         ])
         # Ignore type as floatfmt does allow tuples but mypy does not know that
-        return tabulate(tabular_data, headers=headers,  # type: ignore
+        return tabulate(tabular_data, headers=headers,
-                        floatfmt=floatfmt, tablefmt="pipe")
+                        floatfmt=floatfmt, tablefmt="pipe")  # type: ignore
 
     def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str:
         """
@@ -185,8 +188,8 @@ class Backtesting:
                 len(results[results.profit_abs < 0])
             ])
         # Ignore type as floatfmt does allow tuples but mypy does not know that
-        return tabulate(tabular_data, headers=headers,  # type: ignore
+        return tabulate(tabular_data, headers=headers,
-                        floatfmt=floatfmt, tablefmt="pipe")
+                        floatfmt=floatfmt, tablefmt="pipe")  # type: ignore
 
     def _store_backtest_result(self, recordfilename: Path, results: DataFrame,
                                strategyname: Optional[str] = None) -> None:
@@ -267,6 +270,11 @@ class Backtesting:
                         # - (Expected abs profit + open_rate + open_fee) / (fee_close -1)
                         closerate = - (trade.open_rate * roi + trade.open_rate *
                                        (1 + trade.fee_open)) / (trade.fee_close - 1)
+
+                        # Use the maximum between closerate and low as we
+                        # cannot sell outside of a candle.
+                        # Applies when using {"xx": -1} as roi to force sells after xx minutes
+                        closerate = max(closerate, sell_row.low)
                     else:
                         # This should not be reached...
                         closerate = sell_row.open

freqtrade/optimize/default_hyperopt.py

@@ -11,7 +11,7 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib
 from freqtrade.optimize.hyperopt_interface import IHyperOpt
 
 
-class DefaultHyperOpts(IHyperOpt):
+class DefaultHyperOpt(IHyperOpt):
     """
     Default hyperopt provided by the Freqtrade bot.
     You can override it with your own Hyperopt

freqtrade/optimize/edge_cli.py

@@ -69,8 +69,8 @@ class EdgeCli:
                 ])
 
         # Ignore type as floatfmt does allow tuples but mypy does not know that
-        return tabulate(tabular_data, headers=headers,  # type: ignore
+        return tabulate(tabular_data, headers=headers,
-                        floatfmt=floatfmt, tablefmt="pipe")
+                        floatfmt=floatfmt, tablefmt="pipe")  # type: ignore
 
     def start(self) -> None:
         result = self.edge.calculate()

freqtrade/optimize/hyperopt.py

@@ -98,10 +98,10 @@ class Hyperopt:
         self.position_stacking = self.config.get('position_stacking', False)
 
         if self.has_space('sell'):
-            # Make sure experimental is enabled
+            # Make sure use_sell_signal is enabled
-            if 'experimental' not in self.config:
+            if 'ask_strategy' not in self.config:
-                self.config['experimental'] = {}
+                self.config['ask_strategy'] = {}
-            self.config['experimental']['use_sell_signal'] = True
+            self.config['ask_strategy']['use_sell_signal'] = True
 
     @staticmethod
     def get_lock_filename(config) -> str:

freqtrade/pairlist/IPairList.py

@@ -8,6 +8,9 @@ import logging
 from abc import ABC, abstractmethod
 from typing import List
 
+from freqtrade.exchange import market_is_active
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -77,7 +80,7 @@ class IPairList(ABC):
                 continue
             # Check if market is active
             market = markets[pair]
-            if not market['active']:
+            if not market_is_active(market):
                 logger.info(f"Ignoring {pair} from whitelist. Market is not active.")
                 continue
             sanitized_whitelist.add(pair)

freqtrade/plot/plotting.py

@@ -64,14 +64,13 @@ def add_indicators(fig, row, indicators: List[str], data: pd.DataFrame) -> make_
     """
     for indicator in indicators:
         if indicator in data:
-            # TODO: Figure out why scattergl causes problems
+            scatter = go.Scatter(
-            scattergl = go.Scatter(
                 x=data['date'],
                 y=data[indicator].values,
                 mode='lines',
                 name=indicator
             )
-            fig.add_trace(scattergl, row, 1)
+            fig.add_trace(scatter, row, 1)
         else:
             logger.info(
                 'Indicator "%s" ignored. Reason: This indicator is not found '
@@ -92,7 +91,7 @@ def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> make_sub
     :param name: Name to use
     :return: fig with added profit plot
     """
-    profit = go.Scattergl(
+    profit = go.Scatter(
         x=data.index,
         y=data[column],
         name=name,
@@ -221,23 +220,27 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
         else:
             logger.warning("No sell-signals found.")
 
+    # TODO: Figure out why scattergl causes problems plotly/plotly.js#2284
     if 'bb_lowerband' in data and 'bb_upperband' in data:
-        bb_lower = go.Scattergl(
+        bb_lower = go.Scatter(
             x=data.date,
             y=data.bb_lowerband,
-            name='BB lower',
+            showlegend=False,
             line={'color': 'rgba(255,255,255,0)'},
         )
-        bb_upper = go.Scattergl(
+        bb_upper = go.Scatter(
             x=data.date,
             y=data.bb_upperband,
-            name='BB upper',
+            name='Bollinger Band',
             fill="tonexty",
             fillcolor="rgba(0,176,246,0.2)",
             line={'color': 'rgba(255,255,255,0)'},
         )
         fig.add_trace(bb_lower, 1, 1)
         fig.add_trace(bb_upper, 1, 1)
+        if 'bb_upperband' in indicators1 and 'bb_lowerband' in indicators1:
+            indicators1.remove('bb_upperband')
+            indicators1.remove('bb_lowerband')
 
     # Add indicators to main plot
     fig = add_indicators(fig=fig, row=1, indicators=indicators1, data=data)
@@ -248,26 +251,28 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
     volume = go.Bar(
         x=data['date'],
         y=data['volume'],
-        name='Volume'
+        name='Volume',
-    )
+        marker_color='DarkSlateGrey',
+        marker_line_color='DarkSlateGrey'
+        )
     fig.add_trace(volume, 2, 1)
 
-    # Add indicators to seperate row
+    # Add indicators to separate row
     fig = add_indicators(fig=fig, row=3, indicators=indicators2, data=data)
 
     return fig
 
 
 def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame],
-                          trades: pd.DataFrame) -> go.Figure:
+                          trades: pd.DataFrame, timeframe: str) -> go.Figure:
     # Combine close-values for all pairs, rename columns to "pair"
     df_comb = combine_tickers_with_mean(tickers, "close")
 
     # Add combined cumulative profit
-    df_comb = create_cum_profit(df_comb, trades, 'cum_profit')
+    df_comb = create_cum_profit(df_comb, trades, 'cum_profit', timeframe)
 
     # Plot the pairs average close prices, and total profit growth
-    avgclose = go.Scattergl(
+    avgclose = go.Scatter(
         x=df_comb.index,
         y=df_comb['mean'],
         name='Avg close price',
@@ -288,7 +293,7 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame],
 
     for pair in pairs:
         profit_col = f'cum_profit_{pair}'
-        df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col)
+        df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col, timeframe)
 
         fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}")
 
@@ -377,9 +382,9 @@ def plot_profit(config: Dict[str, Any]) -> None:
                          )
     # Filter trades to relevant pairs
     trades = trades[trades['pair'].isin(plot_elements["pairs"])]
-
     # Create an average close price of all the pairs that were involved.
     # this could be useful to gauge the overall market trend
-    fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades)
+    fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"],
+                                trades, config.get('ticker_interval', '5m'))
     store_plot_file(fig, filename='freqtrade-profit-plot.html',
                     directory=config['user_data_dir'] / "plot", auto_open=True)

freqtrade/resolvers/exchange_resolver.py

@@ -3,7 +3,7 @@ This module loads custom exchanges
 """
 import logging
 
-from freqtrade.exchange import Exchange
+from freqtrade.exchange import Exchange, MAP_EXCHANGE_CHILDCLASS
 import freqtrade.exchange as exchanges
 from freqtrade.resolvers import IResolver
 
@@ -17,19 +17,22 @@ class ExchangeResolver(IResolver):
 
     __slots__ = ['exchange']
 
-    def __init__(self, exchange_name: str, config: dict) -> None:
+    def __init__(self, exchange_name: str, config: dict, validate: bool = True) -> None:
         """
         Load the custom class from config parameter
         :param config: configuration dictionary
         """
+        # Map exchange name to avoid duplicate classes for identical exchanges
+        exchange_name = MAP_EXCHANGE_CHILDCLASS.get(exchange_name, exchange_name)
         exchange_name = exchange_name.title()
         try:
-            self.exchange = self._load_exchange(exchange_name, kwargs={'config': config})
+            self.exchange = self._load_exchange(exchange_name, kwargs={'config': config,
+                                                                       'validate': validate})
         except ImportError:
             logger.info(
                 f"No {exchange_name} specific subclass found. Using the generic class instead.")
         if not hasattr(self, "exchange"):
-            self.exchange = Exchange(config)
+            self.exchange = Exchange(config, validate=validate)
 
     def _load_exchange(
             self, exchange_name: str, kwargs: dict) -> Exchange:
@@ -43,7 +46,7 @@ class ExchangeResolver(IResolver):
         try:
             ex_class = getattr(exchanges, exchange_name)
 
-            exchange = ex_class(kwargs['config'])
+            exchange = ex_class(**kwargs)
             if exchange:
                 logger.info(f"Using resolved exchange '{exchange_name}'...")
                 return exchange

freqtrade/resolvers/hyperopt_resolver.py

@@ -52,14 +52,8 @@ class HyperOptResolver(IResolver):
         """
         current_path = Path(__file__).parent.parent.joinpath('optimize').resolve()
 
-        abs_paths = [
+        abs_paths = self.build_search_paths(config, current_path=current_path,
-            config['user_data_dir'].joinpath('hyperopts'),
+                                            user_subdir='hyperopts', extra_dir=extra_dir)
-            current_path,
-        ]
-
-        if extra_dir:
-            # Add extra hyperopt directory on top of search paths
-            abs_paths.insert(0, Path(extra_dir).resolve())
 
         hyperopt = self._load_object(paths=abs_paths, object_type=IHyperOpt,
                                      object_name=hyperopt_name, kwargs={'config': config})
@@ -109,14 +103,8 @@ class HyperOptLossResolver(IResolver):
         """
         current_path = Path(__file__).parent.parent.joinpath('optimize').resolve()
 
-        abs_paths = [
+        abs_paths = self.build_search_paths(config, current_path=current_path,
-            config['user_data_dir'].joinpath('hyperopts'),
+                                            user_subdir='hyperopts', extra_dir=extra_dir)
-            current_path,
-        ]
-
-        if extra_dir:
-            # Add extra hyperopt directory on top of search paths
-            abs_paths.insert(0, Path(extra_dir).resolve())
 
         hyperoptloss = self._load_object(paths=abs_paths, object_type=IHyperOptLoss,
                                          object_name=hyper_loss_name)

freqtrade/resolvers/iresolver.py

@@ -7,7 +7,7 @@ import importlib.util
 import inspect
 import logging
 from pathlib import Path
-from typing import Any, List, Optional, Tuple, Type, Union
+from typing import Any, List, Optional, Tuple, Union, Generator
 
 logger = logging.getLogger(__name__)
 
@@ -17,15 +17,29 @@ class IResolver:
     This class contains all the logic to load custom classes
     """
 
+    def build_search_paths(self, config, current_path: Path, user_subdir: str,
+                           extra_dir: Optional[str] = None) -> List[Path]:
+
+        abs_paths = [
+            config['user_data_dir'].joinpath(user_subdir),
+            current_path,
+        ]
+
+        if extra_dir:
+            # Add extra directory to the top of the search paths
+            abs_paths.insert(0, Path(extra_dir).resolve())
+
+        return abs_paths
+
     @staticmethod
     def _get_valid_object(object_type, module_path: Path,
-                          object_name: str) -> Optional[Type[Any]]:
+                          object_name: str) -> Generator[Any, None, None]:
         """
-        Returns the first object with matching object_type and object_name in the path given.
+        Generator returning objects with matching object_type and object_name in the path given.
         :param object_type: object_type (class)
         :param module_path: absolute path to the module
         :param object_name: Class name of the object
-        :return: class or None
+        :return: generator containing matching objects
         """
 
         # Generate spec based on absolute path
@@ -42,7 +56,7 @@ class IResolver:
             obj for name, obj in inspect.getmembers(module, inspect.isclass)
             if object_name == name and object_type in obj.__bases__
         )
-        return next(valid_objects_gen, None)
+        return valid_objects_gen
 
     @staticmethod
     def _search_object(directory: Path, object_type, object_name: str,
@@ -59,9 +73,9 @@ class IResolver:
                 logger.debug('Ignoring %s', entry)
                 continue
             module_path = entry.resolve()
-            obj = IResolver._get_valid_object(
+
-                object_type, module_path, object_name
+            obj = next(IResolver._get_valid_object(object_type, module_path, object_name), None)
-            )
+
             if obj:
                 return (obj(**kwargs), module_path)
         return (None, None)

freqtrade/resolvers/pairlist_resolver.py

@@ -1,7 +1,7 @@
 # pragma pylint: disable=attribute-defined-outside-init
 
 """
-This module load custom hyperopts
+This module load custom pairlists
 """
 import logging
 from pathlib import Path
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
 
 class PairListResolver(IResolver):
     """
-    This class contains all the logic to load custom hyperopt class
+    This class contains all the logic to load custom PairList class
     """
 
     __slots__ = ['pairlist']
@@ -39,10 +39,8 @@ class PairListResolver(IResolver):
         """
         current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve()
 
-        abs_paths = [
+        abs_paths = self.build_search_paths(config, current_path=current_path,
-            config['user_data_dir'].joinpath('pairlist'),
+                                            user_subdir='pairlist', extra_dir=None)
-            current_path,
-        ]
 
         pairlist = self._load_object(paths=abs_paths, object_type=IPairList,
                                      object_name=pairlist_name, kwargs=kwargs)

freqtrade/resolvers/strategy_resolver.py

@@ -38,13 +38,13 @@ class StrategyResolver(IResolver):
                                                        config=config,
                                                        extra_dir=config.get('strategy_path'))
 
-        # make sure experimental dict is available
+        # make sure ask_strategy dict is available
-        if 'experimental' not in config:
+        if 'ask_strategy' not in config:
-            config['experimental'] = {}
+            config['ask_strategy'] = {}
 
         # Set attributes
         # Check if we need to override configuration
-        # (Attribute name,                              default, experimental)
+        #             (Attribute name,                    default,     ask_strategy)
         attributes = [("minimal_roi",                     {"0": 10.0}, False),
                       ("ticker_interval",                 None,        False),
                       ("stoploss",                        None,        False),
@@ -57,20 +57,20 @@ class StrategyResolver(IResolver):
                       ("order_time_in_force",             None,        False),
                       ("stake_currency",                  None,        False),
                       ("stake_amount",                    None,        False),
-                      ("use_sell_signal",                 False,       True),
+                      ("use_sell_signal",                 True,        True),
                       ("sell_profit_only",                False,       True),
                       ("ignore_roi_if_buy_signal",        False,       True),
                       ]
-        for attribute, default, experimental in attributes:
+        for attribute, default, ask_strategy in attributes:
-            if experimental:
+            if ask_strategy:
-                self._override_attribute_helper(config['experimental'], attribute, default)
+                self._override_attribute_helper(config['ask_strategy'], attribute, default)
             else:
                 self._override_attribute_helper(config, attribute, default)
 
         # Loop this list again to have output combined
         for attribute, _, exp in attributes:
-            if exp and attribute in config['experimental']:
+            if exp and attribute in config['ask_strategy']:
-                logger.info("Strategy using %s: %s", attribute, config['experimental'][attribute])
+                logger.info("Strategy using %s: %s", attribute, config['ask_strategy'][attribute])
             elif attribute in config:
                 logger.info("Strategy using %s: %s", attribute, config[attribute])
 
@@ -95,7 +95,10 @@ class StrategyResolver(IResolver):
             logger.info("Override strategy '%s' with value in config file: %s.",
                         attribute, config[attribute])
         elif hasattr(self.strategy, attribute):
-            config[attribute] = getattr(self.strategy, attribute)
+            val = getattr(self.strategy, attribute)
+            # None's cannot exist in the config, so do not copy them
+            if val is not None:
+                config[attribute] = val
         # Explicitly check for None here as other "falsy" values are possible
         elif default is not None:
             setattr(self.strategy, attribute, default)
@@ -121,14 +124,8 @@ class StrategyResolver(IResolver):
         """
         current_path = Path(__file__).parent.parent.joinpath('strategy').resolve()
 
-        abs_paths = [
+        abs_paths = self.build_search_paths(config, current_path=current_path,
-            config['user_data_dir'].joinpath('strategies'),
+                                            user_subdir='strategies', extra_dir=extra_dir)
-            current_path,
-        ]
-
-        if extra_dir:
-            # Add extra strategy directory on top of search paths
-            abs_paths.insert(0, Path(extra_dir).resolve())
 
         if ":" in strategy_name:
             logger.info("loading base64 encoded strategy")

freqtrade/rpc/api_server.py

@@ -2,7 +2,7 @@ import logging
 import threading
 from datetime import date, datetime
 from ipaddress import IPv4Address
-from typing import Dict
+from typing import Dict, Callable, Any
 
 from arrow import Arrow
 from flask import Flask, jsonify, request
@@ -34,41 +34,45 @@ class ArrowJSONEncoder(JSONEncoder):
         return JSONEncoder.default(self, obj)
 
 
+# Type should really be Callable[[ApiServer, Any], Any], but that will create a circular dependency
+def require_login(func: Callable[[Any, Any], Any]):
+
+    def func_wrapper(obj, *args, **kwargs):
+
+        auth = request.authorization
+        if auth and obj.check_auth(auth.username, auth.password):
+            return func(obj, *args, **kwargs)
+        else:
+            return jsonify({"error": "Unauthorized"}), 401
+
+    return func_wrapper
+
+
+# Type should really be Callable[[ApiServer], Any], but that will create a circular dependency
+def rpc_catch_errors(func: Callable[[Any], Any]):
+
+    def func_wrapper(obj, *args, **kwargs):
+
+        try:
+            return func(obj, *args, **kwargs)
+        except RPCException as e:
+            logger.exception("API Error calling %s: %s", func.__name__, e)
+            return obj.rest_error(f"Error querying {func.__name__}: {e}")
+
+    return func_wrapper
+
+
 class ApiServer(RPC):
     """
     This class runs api server and provides rpc.rpc functionality to it
 
-    This class starts a none blocking thread the api server runs within
+    This class starts a non-blocking thread the api server runs within
     """
 
-    def rpc_catch_errors(func):
-
-        def func_wrapper(self, *args, **kwargs):
-
-            try:
-                return func(self, *args, **kwargs)
-            except RPCException as e:
-                logger.exception("API Error calling %s: %s", func.__name__, e)
-                return self.rest_error(f"Error querying {func.__name__}: {e}")
-
-        return func_wrapper
-
     def check_auth(self, username, password):
         return (username == self._config['api_server'].get('username') and
                 password == self._config['api_server'].get('password'))
 
-    def require_login(func):
-
-        def func_wrapper(self, *args, **kwargs):
-
-            auth = request.authorization
-            if auth and self.check_auth(auth.username, auth.password):
-                return func(self, *args, **kwargs)
-            else:
-                return jsonify({"error": "Unauthorized"}), 401
-
-        return func_wrapper
-
     def __init__(self, freqtrade) -> None:
         """
         Init the api server, and init the super class RPC

freqtrade/rpc/rpc_manager.py

@@ -18,7 +18,7 @@ class RPCManager:
         self.registered_modules: List[RPC] = []
 
         # Enable telegram
-        if freqtrade.config['telegram'].get('enabled', False):
+        if freqtrade.config.get('telegram', {}).get('enabled', False):
             logger.info('Enabling rpc.telegram ...')
             from freqtrade.rpc.telegram import Telegram
             self.registered_modules.append(Telegram(freqtrade))

freqtrade/strategy/interface.py

@@ -78,8 +78,8 @@ class IStrategy(ABC):
 
     # trailing stoploss
     trailing_stop: bool = False
-    trailing_stop_positive: float
+    trailing_stop_positive: Optional[float] = None
-    trailing_stop_positive_offset: float
+    trailing_stop_positive_offset: float = 0.0
     trailing_only_offset_is_reached = False
 
     # associated ticker interval
@@ -309,9 +309,9 @@ class IStrategy(ABC):
         # Set current rate to high for backtesting sell
         current_rate = high or rate
         current_profit = trade.calc_profit_percent(current_rate)
-        experimental = self.config.get('experimental', {})
+        config_ask_strategy = self.config.get('ask_strategy', {})
 
-        if buy and experimental.get('ignore_roi_if_buy_signal', False):
+        if buy and config_ask_strategy.get('ignore_roi_if_buy_signal', False):
             # This one is noisy, commented out
             # logger.debug(f"{trade.pair} - Buy signal still active. sell_flag=False")
             return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
@@ -322,7 +322,7 @@ class IStrategy(ABC):
                          f"sell_type=SellType.ROI")
             return SellCheckTuple(sell_flag=True, sell_type=SellType.ROI)
 
-        if experimental.get('sell_profit_only', False):
+        if config_ask_strategy.get('sell_profit_only', False):
             # This one is noisy, commented out
             # logger.debug(f"{trade.pair} - Checking if trade is profitable...")
             if trade.calc_profit(rate=rate) <= 0:
@@ -330,7 +330,7 @@ class IStrategy(ABC):
                 # logger.debug(f"{trade.pair} - Trade is not profitable. sell_flag=False")
                 return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
 
-        if sell and not buy and experimental.get('use_sell_signal', False):
+        if sell and not buy and config_ask_strategy.get('use_sell_signal', True):
             logger.debug(f"{trade.pair} - Sell signal received. sell_flag=True, "
                          f"sell_type=SellType.SELL_SIGNAL")
             return SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL)
@@ -347,26 +347,23 @@ class IStrategy(ABC):
         decides to sell or not
         :param current_profit: current profit in percent
         """
-        trailing_stop = self.config.get('trailing_stop', False)
         stop_loss_value = force_stoploss if force_stoploss else self.stoploss
 
         # Initiate stoploss with open_rate. Does nothing if stoploss is already set.
         trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True)
 
-        if trailing_stop:
+        if self.trailing_stop:
             # trailing stoploss handling
-            sl_offset = self.config.get('trailing_stop_positive_offset') or 0.0
+            sl_offset = self.trailing_stop_positive_offset
-            tsl_only_offset = self.config.get('trailing_only_offset_is_reached', False)
 
             # Make sure current_profit is calculated using high for backtesting.
             high_profit = current_profit if not high else trade.calc_profit_percent(high)
 
             # Don't update stoploss if trailing_only_offset_is_reached is true.
-            if not (tsl_only_offset and high_profit < sl_offset):
+            if not (self.trailing_only_offset_is_reached and high_profit < sl_offset):
                 # Specific handling for trailing_stop_positive
-                if 'trailing_stop_positive' in self.config and high_profit > sl_offset:
+                if self.trailing_stop_positive is not None and high_profit > sl_offset:
-                    # Ignore mypy error check in configuration that this is a float
+                    stop_loss_value = self.trailing_stop_positive
-                    stop_loss_value = self.config.get('trailing_stop_positive')  # type: ignore
                     logger.debug(f"{trade.pair} - Using positive stoploss: {stop_loss_value} "
                                  f"offset: {sl_offset:.4g} profit: {current_profit:.4f}%")
 

freqtrade/utils.py

@@ -1,15 +1,23 @@
 import logging
 import sys
+from collections import OrderedDict
 from pathlib import Path
 from typing import Any, Dict, List
 
 import arrow
+import csv
+import rapidjson
+from tabulate import tabulate
 
 from freqtrade import OperationalException
 from freqtrade.configuration import Configuration, TimeRange
 from freqtrade.configuration.directory_operations import create_userdata_dir
-from freqtrade.data.history import refresh_backtest_ohlcv_data
+from freqtrade.data.history import (convert_trades_to_ohlcv,
-from freqtrade.exchange import available_exchanges
+                                    refresh_backtest_ohlcv_data,
+                                    refresh_backtest_trades_data)
+from freqtrade.exchange import (available_exchanges, ccxt_exchanges, market_is_active,
+                                symbol_is_pair)
+from freqtrade.misc import plural
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.state import RunMode
 
@@ -39,12 +47,14 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
     :param args: Cli args from Arguments()
     :return: None
     """
-
+    exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges()
     if args['print_one_column']:
-        print('\n'.join(available_exchanges()))
+        print('\n'.join(exchanges))
     else:
-        print(f"Exchanges supported by ccxt and available for Freqtrade: "
+        if args['list_exchanges_all']:
-              f"{', '.join(available_exchanges())}")
+            print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}")
+        else:
+            print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}")
 
 
 def start_create_userdir(args: Dict[str, Any]) -> None:
@@ -82,13 +92,23 @@ def start_download_data(args: Dict[str, Any]) -> None:
 
     pairs_not_available: List[str] = []
 
+    # Init exchange
+    exchange = ExchangeResolver(config['exchange']['name'], config).exchange
     try:
-        # Init exchange
-        exchange = ExchangeResolver(config['exchange']['name'], config).exchange
 
-        pairs_not_available = refresh_backtest_ohlcv_data(
+        if config.get('download_trades'):
-            exchange, pairs=config["pairs"], timeframes=config["timeframes"],
+            pairs_not_available = refresh_backtest_trades_data(
-            dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
+                exchange, pairs=config["pairs"], datadir=Path(config['datadir']),
+                timerange=timerange, erase=config.get("erase"))
+
+            # Convert downloaded trade data to different timeframes
+            convert_trades_to_ohlcv(
+                pairs=config["pairs"], timeframes=config["timeframes"],
+                datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
+        else:
+            pairs_not_available = refresh_backtest_ohlcv_data(
+                exchange, pairs=config["pairs"], timeframes=config["timeframes"],
+                dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
 
     except KeyboardInterrupt:
         sys.exit("SIGINT received, aborting ...")
@@ -96,4 +116,106 @@ def start_download_data(args: Dict[str, Any]) -> None:
     finally:
         if pairs_not_available:
             logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
-                        f"on exchange {config['exchange']['name']}.")
+                        f"on exchange {exchange.name}.")
+
+
+def start_list_timeframes(args: Dict[str, Any]) -> None:
+    """
+    Print ticker intervals (timeframes) available on Exchange
+    """
+    config = setup_utils_configuration(args, RunMode.OTHER)
+    # Do not use ticker_interval set in the config
+    config['ticker_interval'] = None
+
+    # Init exchange
+    exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange
+
+    if args['print_one_column']:
+        print('\n'.join(exchange.timeframes))
+    else:
+        print(f"Timeframes available for the exchange `{exchange.name}`: "
+              f"{', '.join(exchange.timeframes)}")
+
+
+def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
+    """
+    Print pairs/markets on the exchange
+    :param args: Cli args from Arguments()
+    :param pairs_only: if True print only pairs, otherwise print all instruments (markets)
+    :return: None
+    """
+    config = setup_utils_configuration(args, RunMode.OTHER)
+
+    # Init exchange
+    exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange
+
+    # By default only active pairs/markets are to be shown
+    active_only = not args.get('list_pairs_all', False)
+
+    base_currencies = args.get('base_currencies', [])
+    quote_currencies = args.get('quote_currencies', [])
+
+    try:
+        pairs = exchange.get_markets(base_currencies=base_currencies,
+                                     quote_currencies=quote_currencies,
+                                     pairs_only=pairs_only,
+                                     active_only=active_only)
+        # Sort the pairs/markets by symbol
+        pairs = OrderedDict(sorted(pairs.items()))
+    except Exception as e:
+        raise OperationalException(f"Cannot get markets. Reason: {e}") from e
+
+    else:
+        summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") +
+                       ("active " if active_only else "") +
+                       (plural(len(pairs), "pair" if pairs_only else "market")) +
+                       (f" with {', '.join(base_currencies)} as base "
+                        f"{plural(len(base_currencies), 'currency', 'currencies')}"
+                        if base_currencies else "") +
+                       (" and" if base_currencies and quote_currencies else "") +
+                       (f" with {', '.join(quote_currencies)} as quote "
+                        f"{plural(len(quote_currencies), 'currency', 'currencies')}"
+                        if quote_currencies else ""))
+
+        headers = ["Id", "Symbol", "Base", "Quote", "Active",
+                   *(['Is pair'] if not pairs_only else [])]
+
+        tabular_data = []
+        for _, v in pairs.items():
+            tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'],
+                                 'Base': v['base'], 'Quote': v['quote'],
+                                 'Active': market_is_active(v),
+                                 **({'Is pair': symbol_is_pair(v['symbol'])}
+                                    if not pairs_only else {})})
+
+        if (args.get('print_one_column', False) or
+                args.get('list_pairs_print_json', False) or
+                args.get('print_csv', False)):
+            # Print summary string in the log in case of machine-readable
+            # regular formats.
+            logger.info(f"{summary_str}.")
+        else:
+            # Print empty string separating leading logs and output in case of
+            # human-readable formats.
+            print()
+
+        if len(pairs):
+            if args.get('print_list', False):
+                # print data as a list, with human-readable summary
+                print(f"{summary_str}: {', '.join(pairs.keys())}.")
+            elif args.get('print_one_column', False):
+                print('\n'.join(pairs.keys()))
+            elif args.get('list_pairs_print_json', False):
+                print(rapidjson.dumps(list(pairs.keys()), default=str))
+            elif args.get('print_csv', False):
+                writer = csv.DictWriter(sys.stdout, fieldnames=headers)
+                writer.writeheader()
+                writer.writerows(tabular_data)
+            else:
+                # print data as a table, with the human-readable summary
+                print(f"{summary_str}:")
+                print(tabulate(tabular_data, headers='keys', tablefmt='pipe'))
+        elif not (args.get('print_one_column', False) or
+                  args.get('list_pairs_print_json', False) or
+                  args.get('print_csv', False)):
+            print(f"{summary_str}.")

mkdocs.yml

@@ -14,14 +14,17 @@ nav:
     - Data Downloading: data-download.md
     - Backtesting: backtesting.md
     - Hyperopt: hyperopt.md
-    - Edge positioning: edge.md
+    - Edge Positioning: edge.md
+    - Utility Subcommands: utils.md
     - FAQ: faq.md
-    - Data Analysis: data-analysis.md
+    - Data Analysis:
-    - Plotting: plotting.md
+        - Jupyter Notebooks: data-analysis.md
+        - Strategy analysis: strategy_analysis_example.md
+        - Plotting: plotting.md
     - SQL Cheatsheet: sql_cheatsheet.md
-    - Sandbox testing: sandbox-testing.md
+    - Sandbox Testing: sandbox-testing.md
-    - Deprecated features: deprecated.md
+    - Deprecated Features: deprecated.md
-    - Contributors guide: developer.md
+    - Contributors Guide: developer.md
 theme:
   name: material
   logo: 'images/logo.png'
@@ -29,6 +32,8 @@ theme:
   palette:
     primary: 'blue grey'
     accent: 'tear'
+extra_css:
+  - 'stylesheets/ft.extra.css'
 markdown_extensions:
     - admonition
     - codehilite:
@@ -47,3 +52,4 @@ markdown_extensions:
     - pymdownx.tasklist:
         custom_checkbox: true
     - pymdownx.tilde
+    - mdx_truly_sane_lists

requirements-common.txt

@@ -1,16 +1,16 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.18.1180
+ccxt==1.18.1306
-SQLAlchemy==1.3.8
+SQLAlchemy==1.3.10
-python-telegram-bot==12.1.1
+python-telegram-bot==12.2.0
 arrow==0.15.2
 cachetools==3.1.1
 requests==2.22.0
-urllib3==1.25.5
+urllib3==1.25.6
 wrapt==1.11.2
-jsonschema==3.0.2
+jsonschema==3.1.1
 TA-Lib==0.4.17
-tabulate==0.8.3
+tabulate==0.8.5
 coinmarketcap==5.0.3
 
 # find first, C search in arrays

requirements-dev.txt

@@ -6,10 +6,13 @@
 coveralls==1.8.2
 flake8==3.7.8
 flake8-type-annotations==0.1.0
-flake8-tidy-imports==2.0.0
+flake8-tidy-imports==3.0.0
-mypy==0.720
+mypy==0.740
-pytest==5.1.3
+pytest==5.2.1
 pytest-asyncio==0.10.0
-pytest-cov==2.7.1
+pytest-cov==2.8.1
-pytest-mock==1.10.4
+pytest-mock==1.11.1
 pytest-random-order==1.0.4
+
+# Convert jupyter notebooks to markdown documents
+nbconvert==5.6.0

requirements-hyperopt.txt

@@ -1,9 +1,9 @@
 # Include all requirements to run the bot.
-# -r requirements.txt
+-r requirements.txt
 
 # Required for hyperopt
 scipy==1.3.1
 scikit-learn==0.21.3
 scikit-optimize==0.5.2
 filelock==3.0.12
-joblib==0.13.2
+joblib==0.14.0

requirements-plot.txt

@@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==4.1.1
+plotly==4.2.1
 

requirements.txt

@@ -1,5 +1,5 @@
 # Load common requirements
 -r requirements-common.txt
 
-numpy==1.17.2
+numpy==1.17.3
-pandas==0.25.1
+pandas==0.25.2

scripts/download_backtest_data.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-
-
-print("This script has been integrated into freqtrade "
-      "and its functionality is available by calling `freqtrade download-data`.")
-print("Please check the documentation on https://www.freqtrade.io/en/latest/backtesting/ "
-      "for details.")
-
-sys.exit(1)

scripts/get_market_pairs.py

@@ -1,103 +0,0 @@
-"""
-This script was adapted from ccxt here:
-https://github.com/ccxt/ccxt/blob/master/examples/py/arbitrage-pairs.py
-"""
-import os
-import sys
-import traceback
-
-root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.append(root + '/python')
-
-import ccxt  # noqa: E402
-
-
-def style(s, style):
-    return style + s + '\033[0m'
-
-
-def green(s):
-    return style(s, '\033[92m')
-
-
-def blue(s):
-    return style(s, '\033[94m')
-
-
-def yellow(s):
-    return style(s, '\033[93m')
-
-
-def red(s):
-    return style(s, '\033[91m')
-
-
-def pink(s):
-    return style(s, '\033[95m')
-
-
-def bold(s):
-    return style(s, '\033[1m')
-
-
-def underline(s):
-    return style(s, '\033[4m')
-
-
-def dump(*args):
-    print(' '.join([str(arg) for arg in args]))
-
-
-def print_supported_exchanges():
-    dump('Supported exchanges:', green(', '.join(ccxt.exchanges)))
-
-
-try:
-
-    if len(sys.argv) < 2:
-        dump("Usage: python " + sys.argv[0], green('id'))
-        print_supported_exchanges()
-        sys.exit(1)
-
-    id = sys.argv[1]  # get exchange id from command line arguments
-
-    # check if the exchange is supported by ccxt
-    exchange_found = id in ccxt.exchanges
-
-    if exchange_found:
-        dump('Instantiating', green(id), 'exchange')
-
-        # instantiate the exchange by id
-        exchange = getattr(ccxt, id)({
-            # 'proxy':'https://cors-anywhere.herokuapp.com/',
-        })
-
-        # load all markets from the exchange
-        markets = exchange.load_markets()
-
-        # output a list of all market symbols
-        dump(green(id), 'has', len(exchange.symbols), 'symbols:', exchange.symbols)
-
-        tuples = list(ccxt.Exchange.keysort(markets).items())
-
-        # debug
-        for (k, v) in tuples:
-            print(v)
-
-        # output a table of all markets
-        dump(pink('{:<15} {:<15} {:<15} {:<15}'.format('id', 'symbol', 'base', 'quote')))
-
-        for (k, v) in tuples:
-            dump('{:<15} {:<15} {:<15} {:<15}'.format(v['id'], v['symbol'], v['base'], v['quote']))
-
-    else:
-
-        dump('Exchange ' + red(id) + ' not found')
-        print_supported_exchanges()
-
-except Exception as e:
-    dump('[' + type(e).__name__ + ']', str(e))
-    dump(traceback.format_exc())
-    dump("Usage: python " + sys.argv[0], green('id'))
-    print_supported_exchanges()
-    sys.exit(1)

scripts/plot_dataframe.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-
-
-print("This script has been integrated into freqtrade "
-      "and its functionality is available by calling `freqtrade plot-dataframe`.")
-print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ "
-      "for details.")
-
-sys.exit(1)

scripts/plot_profit.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-
-
-print("This script has been integrated into freqtrade "
-      "and its functionality is available by calling `freqtrade plot-profit`.")
-print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ "
-      "for details.")
-
-sys.exit(1)

setup.py

@@ -43,6 +43,7 @@ jupyter = [
     'jupyter',
     'nbstripout',
     'ipykernel',
+    'nbconvert',
     ]
 
 all_extra = api + plot + develop + jupyter + hyperopt

tests/config_test_comments.json

@@ -78,7 +78,7 @@
             "ZEC/BTC",
             "XLM/BTC",
             "NXT/BTC",
-            "POWR/BTC",
+            "TRX/BTC",
             "ADA/BTC",
             "XMR/BTC"
         ],
@@ -103,11 +103,6 @@
         "max_trade_duration_minute": 1440,
         "remove_pumps": false
     },
-    "experimental": {
-        "use_sell_signal": false,
-        "sell_profit_only": false,
-        "ignore_roi_if_buy_signal": false
-    },
     "telegram": {
 // We can now comment out some settings
 //        "enabled": true,

tests/conftest.py

@@ -9,8 +9,8 @@ from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
 
 import arrow
-import pytest
 import numpy as np
+import pytest
 from telegram import Chat, Message, Update
 
 from freqtrade import constants, persistence
@@ -19,10 +19,10 @@ from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.edge import Edge, PairInfo
 from freqtrade.exchange import Exchange
 from freqtrade.freqtradebot import FreqtradeBot
+from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.worker import Worker
 
-
 logging.getLogger('').setLevel(logging.INFO)
 
 
@@ -318,7 +318,8 @@ def markets():
             'symbol': 'TKN/BTC',
             'base': 'TKN',
             'quote': 'BTC',
-            'active': True,
+            # According to ccxt, markets without active item set are also active
+            # 'active': True,
             'precision': {
                 'price': 8,
                 'amount': 8,
@@ -509,6 +510,50 @@ def markets():
                 }
             },
             'info': {},
+        },
+        'LTC/USD': {
+            'id': 'USD-LTC',
+            'symbol': 'LTC/USD',
+            'base': 'LTC',
+            'quote': 'USD',
+            'active': True,
+            'precision': {
+                'amount': 8,
+                'price': 8
+            },
+            'limits': {
+                'amount': {
+                    'min': 0.06646786,
+                    'max': None
+                },
+                'price': {
+                    'min': 1e-08,
+                    'max': None
+                }
+            },
+            'info': {},
+        },
+        'XLTCUSDT': {
+            'id': 'xLTCUSDT',
+            'symbol': 'XLTCUSDT',
+            'base': 'LTC',
+            'quote': 'USDT',
+            'active': True,
+            'precision': {
+                'amount': 8,
+                'price': 8
+            },
+            'limits': {
+                'amount': {
+                    'min': 0.06646786,
+                    'max': None
+                },
+                'price': {
+                    'min': 1e-08,
+                    'max': None
+                }
+            },
+            'info': {},
         }
     }
 
@@ -608,6 +653,14 @@ def limit_buy_order_old_partial():
     }
 
 
+@pytest.fixture
+def limit_buy_order_old_partial_canceled(limit_buy_order_old_partial):
+    res = deepcopy(limit_buy_order_old_partial)
+    res['status'] = 'canceled'
+    res['fee'] = {'cost': 0.0001, 'currency': 'ETH'}
+    return res
+
+
 @pytest.fixture
 def limit_sell_order():
     return {
@@ -896,12 +949,6 @@ def result(testdatadir):
         return parse_ticker_dataframe(json.load(data_file), '1m', pair="UNITTEST/BTC",
                                       fill_missing=True)
 
-# FIX:
-# Create an fixture/function
-# that inserts a trade of some type and open-status
-# return the open-order-id
-# See tests in rpc/main that could use this
-
 
 @pytest.fixture(scope="function")
 def trades_for_order():
@@ -928,6 +975,110 @@ def trades_for_order():
              'fee': {'cost': 0.008, 'currency': 'LTC'}}]
 
 
+@pytest.fixture(scope="function")
+def trades_history():
+    return [{'info': {'a': 126181329,
+                      'p': '0.01962700',
+                      'q': '0.04000000',
+                      'f': 138604155,
+                      'l': 138604155,
+                      'T': 1565798399463,
+                      'm': False,
+                      'M': True},
+             'timestamp': 1565798399463,
+             'datetime': '2019-08-14T15:59:59.463Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181329',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'buy',
+             'price': 0.019627,
+             'amount': 0.04,
+             'cost': 0.00078508,
+             'fee': None},
+            {'info': {'a': 126181330,
+                      'p': '0.01962700',
+                      'q': '0.24400000',
+                      'f': 138604156,
+                      'l': 138604156,
+                      'T': 1565798399629,
+                      'm': False,
+                      'M': True},
+             'timestamp': 1565798399629,
+             'datetime': '2019-08-14T15:59:59.629Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181330',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'buy',
+             'price': 0.019627,
+             'amount': 0.244,
+             'cost': 0.004788987999999999,
+             'fee': None},
+            {'info': {'a': 126181331,
+                      'p': '0.01962600',
+                      'q': '0.01100000',
+                      'f': 138604157,
+                      'l': 138604157,
+                      'T': 1565798399752,
+                      'm': True,
+                      'M': True},
+             'timestamp': 1565798399752,
+             'datetime': '2019-08-14T15:59:59.752Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181331',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'sell',
+             'price': 0.019626,
+             'amount': 0.011,
+             'cost': 0.00021588599999999999,
+             'fee': None},
+            {'info': {'a': 126181332,
+                      'p': '0.01962600',
+                      'q': '0.01100000',
+                      'f': 138604158,
+                      'l': 138604158,
+                      'T': 1565798399862,
+                      'm': True,
+                      'M': True},
+             'timestamp': 1565798399862,
+             'datetime': '2019-08-14T15:59:59.862Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181332',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'sell',
+             'price': 0.019626,
+             'amount': 0.011,
+             'cost': 0.00021588599999999999,
+             'fee': None},
+            {'info': {'a': 126181333,
+                      'p': '0.01952600',
+                      'q': '0.01200000',
+                      'f': 138604158,
+                      'l': 138604158,
+                      'T': 1565798399872,
+                      'm': True,
+                      'M': True},
+             'timestamp': 1565798399872,
+             'datetime': '2019-08-14T15:59:59.872Z',
+             'symbol': 'ETH/BTC',
+             'id': '126181333',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'sell',
+             'price': 0.019626,
+             'amount': 0.011,
+             'cost': 0.00021588599999999999,
+             'fee': None}]
+
+
 @pytest.fixture(scope="function")
 def trades_for_order2():
     return [{'info': {'id': 34567,
@@ -1075,3 +1226,19 @@ def import_fails() -> None:
 
     # restore previous importfunction
     builtins.__import__ = realimport
+
+
+@pytest.fixture(scope="function")
+def open_trade():
+    return Trade(
+        pair='ETH/BTC',
+        open_rate=0.00001099,
+        exchange='bittrex',
+        open_order_id='123456789',
+        amount=90.99181073,
+        fee_open=0.0,
+        fee_close=0.0,
+        stake_amount=1,
+        open_date=arrow.utcnow().shift(minutes=-601).datetime,
+        is_open=True
+    )

tests/data/test_btanalysis.py

@@ -2,7 +2,7 @@ from unittest.mock import MagicMock
 
 import pytest
 from arrow import Arrow
-from pandas import DataFrame, to_datetime
+from pandas import DataFrame, DateOffset, to_datetime
 
 from freqtrade.configuration import TimeRange
 from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
@@ -53,12 +53,12 @@ def test_load_trades_db(default_conf, fee, mocker):
 
 def test_extract_trades_of_period(testdatadir):
     pair = "UNITTEST/BTC"
-    timerange = TimeRange(None, 'line', 0, -1000)
+    # 2018-11-14 06:07:00
+    timerange = TimeRange('date', None, 1510639620, 0)
 
     data = load_pair_history(pair=pair, ticker_interval='1m',
                              datadir=testdatadir, timerange=timerange)
 
-    # timerange = 2017-11-14 06:07 - 2017-11-14 22:58:00
     trades = DataFrame(
         {'pair': [pair, pair, pair, pair],
          'profit_percent': [0.0, 0.1, -0.2, -0.5],
@@ -108,7 +108,7 @@ def test_load_trades(default_conf, mocker):
 
 
 def test_combine_tickers_with_mean(testdatadir):
-    pairs = ["ETH/BTC", "XLM/BTC"]
+    pairs = ["ETH/BTC", "ADA/BTC"]
     tickers = load_data(datadir=testdatadir,
                         pairs=pairs,
                         ticker_interval='5m'
@@ -116,7 +116,7 @@ def test_combine_tickers_with_mean(testdatadir):
     df = combine_tickers_with_mean(tickers)
     assert isinstance(df, DataFrame)
     assert "ETH/BTC" in df.columns
-    assert "XLM/BTC" in df.columns
+    assert "ADA/BTC" in df.columns
     assert "mean" in df.columns
 
 
@@ -125,12 +125,30 @@ def test_create_cum_profit(testdatadir):
     bt_data = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
-    df = load_pair_history(pair="POWR/BTC", ticker_interval='5m',
+    df = load_pair_history(pair="TRX/BTC", ticker_interval='5m',
                            datadir=testdatadir, timerange=timerange)
 
     cum_profits = create_cum_profit(df.set_index('date'),
-                                    bt_data[bt_data["pair"] == 'POWR/BTC'],
+                                    bt_data[bt_data["pair"] == 'TRX/BTC'],
-                                    "cum_profits")
+                                    "cum_profits", timeframe="5m")
+    assert "cum_profits" in cum_profits.columns
+    assert cum_profits.iloc[0]['cum_profits'] == 0
+    assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005
+
+
+def test_create_cum_profit1(testdatadir):
+    filename = testdatadir / "backtest-result_test.json"
+    bt_data = load_backtest_data(filename)
+    # Move close-time to "off" the candle, to make sure the logic still works
+    bt_data.loc[:, 'close_time'] = bt_data.loc[:, 'close_time'] + DateOffset(seconds=20)
+    timerange = TimeRange.parse_timerange("20180110-20180112")
+
+    df = load_pair_history(pair="TRX/BTC", ticker_interval='5m',
+                           datadir=testdatadir, timerange=timerange)
+
+    cum_profits = create_cum_profit(df.set_index('date'),
+                                    bt_data[bt_data["pair"] == 'TRX/BTC'],
+                                    "cum_profits", timeframe="5m")
     assert "cum_profits" in cum_profits.columns
     assert cum_profits.iloc[0]['cum_profits'] == 0
     assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005

tests/data/test_dataprovider.py

@@ -120,3 +120,35 @@ def test_refresh(mocker, default_conf, ticker_history):
     assert len(refresh_mock.call_args[0]) == 1
     assert len(refresh_mock.call_args[0][0]) == len(pairs) + len(pairs_non_trad)
     assert refresh_mock.call_args[0][0] == pairs + pairs_non_trad
+
+
+def test_orderbook(mocker, default_conf, order_book_l2):
+    api_mock = MagicMock()
+    api_mock.fetch_l2_order_book = order_book_l2
+    exchange = get_patched_exchange(mocker, default_conf, api_mock=api_mock)
+
+    dp = DataProvider(default_conf, exchange)
+    res = dp.orderbook('ETH/BTC', 5)
+    assert order_book_l2.call_count == 1
+    assert order_book_l2.call_args_list[0][0][0] == 'ETH/BTC'
+    assert order_book_l2.call_args_list[0][0][1] == 5
+
+    assert type(res) is dict
+    assert 'bids' in res
+    assert 'asks' in res
+
+
+def test_market(mocker, default_conf, markets):
+    api_mock = MagicMock()
+    api_mock.markets = markets
+    exchange = get_patched_exchange(mocker, default_conf, api_mock=api_mock)
+
+    dp = DataProvider(default_conf, exchange)
+    res = dp.market('ETH/BTC')
+
+    assert type(res) is dict
+    assert 'symbol' in res
+    assert res['symbol'] == 'ETH/BTC'
+
+    res = dp.market('UNITTEST/BTC')
+    assert res is None

tests/data/test_history.py

@@ -1,7 +1,6 @@
 # pragma pylint: disable=missing-docstring, protected-access, C0103
 
 import json
-import os
 import uuid
 from pathlib import Path
 from shutil import copyfile
@@ -14,49 +13,54 @@ from pandas import DataFrame
 from freqtrade import OperationalException
 from freqtrade.configuration import TimeRange
 from freqtrade.data import history
-from freqtrade.data.history import (download_pair_history,
+from freqtrade.data.history import (_load_cached_data_for_updating,
-                                    load_cached_data_for_updating,
+                                    convert_trades_to_ohlcv,
-                                    load_tickerdata_file,
+                                    download_pair_history,
+                                    download_trades_history,
+                                    load_tickerdata_file, pair_data_filename,
+                                    pair_trades_filename,
                                     refresh_backtest_ohlcv_data,
+                                    refresh_backtest_trades_data,
                                     trim_tickerlist)
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
 from freqtrade.strategy.default_strategy import DefaultStrategy
-from tests.conftest import get_patched_exchange, log_has, log_has_re, patch_exchange
+from tests.conftest import (get_patched_exchange, log_has, log_has_re,
+                            patch_exchange)
 
 # Change this if modifying UNITTEST/BTC testdatafile
 _BTC_UNITTEST_LENGTH = 13681
 
 
-def _backup_file(file: str, copy_file: bool = False) -> None:
+def _backup_file(file: Path, copy_file: bool = False) -> None:
     """
     Backup existing file to avoid deleting the user file
     :param file: complete path to the file
     :param touch_file: create an empty file in replacement
     :return: None
     """
-    file_swp = file + '.swp'
+    file_swp = str(file) + '.swp'
-    if os.path.isfile(file):
+    if file.is_file():
-        os.rename(file, file_swp)
+        file.rename(file_swp)
 
         if copy_file:
             copyfile(file_swp, file)
 
 
-def _clean_test_file(file: str) -> None:
+def _clean_test_file(file: Path) -> None:
     """
     Backup existing file to avoid deleting the user file
     :param file: complete path to the file
     :return: None
     """
-    file_swp = file + '.swp'
+    file_swp = Path(str(file) + '.swp')
     # 1. Delete file from the test
-    if os.path.isfile(file):
+    if file.is_file():
-        os.remove(file)
+        file.unlink()
 
     # 2. Rollback to the initial file
-    if os.path.isfile(file_swp):
+    if file_swp.is_file():
-        os.rename(file_swp, file)
+        file_swp.rename(file)
 
 
 def test_load_data_30min_ticker(mocker, caplog, default_conf, testdatadir) -> None:
@@ -80,10 +84,10 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> Non
 
 def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> None:
     mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history)
-    file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json')
+    file = testdatadir / 'UNITTEST_BTC-1m.json'
     _backup_file(file, copy_file=True)
     history.load_data(datadir=testdatadir, ticker_interval='1m', pairs=['UNITTEST/BTC'])
-    assert os.path.isfile(file) is True
+    assert file.is_file()
     assert not log_has(
         'Download history data for pair: "UNITTEST/BTC", interval: 1m '
         'and store in None.', caplog
@@ -98,14 +102,14 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
     """
     mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list)
     exchange = get_patched_exchange(mocker, default_conf)
-    file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
+    file = testdatadir / 'MEME_BTC-1m.json'
 
     _backup_file(file)
     # do not download a new pair if refresh_pairs isn't set
     history.load_pair_history(datadir=testdatadir,
                               ticker_interval='1m',
                               pair='MEME/BTC')
-    assert os.path.isfile(file) is False
+    assert not file.is_file()
     assert log_has(
         'No history data for pair: "MEME/BTC", interval: 1m. '
         'Use `freqtrade download-data` to download the data', caplog
@@ -117,7 +121,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
                               refresh_pairs=True,
                               exchange=exchange,
                               pair='MEME/BTC')
-    assert os.path.isfile(file) is True
+    assert file.is_file()
     assert log_has_re(
         'Download history data for pair: "MEME/BTC", interval: 1m '
         'and store in .*', caplog
@@ -135,6 +139,18 @@ def test_testdata_path(testdatadir) -> None:
     assert str(Path('tests') / 'testdata') in str(testdatadir)
 
 
+def test_pair_data_filename():
+    fn = pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m')
+    assert isinstance(fn, Path)
+    assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json')
+
+
+def test_pair_trades_filename():
+    fn = pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC')
+    assert isinstance(fn, Path)
+    assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json.gz')
+
+
 def test_load_cached_data_for_updating(mocker) -> None:
     datadir = Path(__file__).parent.parent.joinpath('testdata')
 
@@ -151,43 +167,43 @@ def test_load_cached_data_for_updating(mocker) -> None:
     # timeframe starts earlier than the cached data
     # should fully update data
     timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0)
-    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == []
     assert start_ts == test_data[0][0] - 1000
 
     # same with 'line' timeframe
     num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120
-    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m',
+    data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m',
-                                                   TimeRange(None, 'line', 0, -num_lines))
+                                                    TimeRange(None, 'line', 0, -num_lines))
     assert data == []
     assert start_ts < test_data[0][0] - 1
 
     # timeframe starts in the center of the cached data
     # should return the chached data w/o the last item
     timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0)
-    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
     # same with 'line' timeframe
     num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30
     timerange = TimeRange(None, 'line', 0, -num_lines)
-    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
     # timeframe starts after the chached data
     # should return the chached data w/o the last item
     timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0)
-    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
     # Try loading last 30 lines.
-    # Not supported by load_cached_data_for_updating, we always need to get the full data.
+    # Not supported by _load_cached_data_for_updating, we always need to get the full data.
     num_lines = 30
     timerange = TimeRange(None, 'line', 0, -num_lines)
-    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
@@ -195,27 +211,27 @@ def test_load_cached_data_for_updating(mocker) -> None:
     # should return the chached data w/o the last item
     num_lines = 30
     timerange = TimeRange(None, 'line', 0, -num_lines)
-    data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
+    data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange)
     assert data == test_data[:-1]
     assert test_data[-2][0] < start_ts < test_data[-1][0]
 
     # no datafile exist
     # should return timestamp start time
     timerange = TimeRange('date', None, now_ts - 10000, 0)
-    data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange)
+    data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange)
     assert data == []
     assert start_ts == (now_ts - 10000) * 1000
 
     # same with 'line' timeframe
     num_lines = 30
     timerange = TimeRange(None, 'line', 0, -num_lines)
-    data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange)
+    data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange)
     assert data == []
     assert start_ts == (now_ts - num_lines * 60) * 1000
 
     # no datafile exist, no timeframe is set
     # should return an empty array and None
-    data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None)
+    data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None)
     assert data == []
     assert start_ts is None
 
@@ -223,18 +239,18 @@ def test_load_cached_data_for_updating(mocker) -> None:
 def test_download_pair_history(ticker_history_list, mocker, default_conf, testdatadir) -> None:
     mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list)
     exchange = get_patched_exchange(mocker, default_conf)
-    file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
+    file1_1 = testdatadir / 'MEME_BTC-1m.json'
-    file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json')
+    file1_5 = testdatadir / 'MEME_BTC-5m.json'
-    file2_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-1m.json')
+    file2_1 = testdatadir / 'CFI_BTC-1m.json'
-    file2_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-5m.json')
+    file2_5 = testdatadir / 'CFI_BTC-5m.json'
 
     _backup_file(file1_1)
     _backup_file(file1_5)
     _backup_file(file2_1)
     _backup_file(file2_5)
 
-    assert os.path.isfile(file1_1) is False
+    assert not file1_1.is_file()
-    assert os.path.isfile(file2_1) is False
+    assert not file2_1.is_file()
 
     assert download_pair_history(datadir=testdatadir, exchange=exchange,
                                  pair='MEME/BTC',
@@ -243,15 +259,15 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda
                                  pair='CFI/BTC',
                                  ticker_interval='1m')
     assert not exchange._pairs_last_refresh_time
-    assert os.path.isfile(file1_1) is True
+    assert file1_1.is_file()
-    assert os.path.isfile(file2_1) is True
+    assert file2_1.is_file()
 
     # clean files freshly downloaded
     _clean_test_file(file1_1)
     _clean_test_file(file2_1)
 
-    assert os.path.isfile(file1_5) is False
+    assert not file1_5.is_file()
-    assert os.path.isfile(file2_5) is False
+    assert not file2_5.is_file()
 
     assert download_pair_history(datadir=testdatadir, exchange=exchange,
                                  pair='MEME/BTC',
@@ -260,8 +276,8 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda
                                  pair='CFI/BTC',
                                  ticker_interval='5m')
     assert not exchange._pairs_last_refresh_time
-    assert os.path.isfile(file1_5) is True
+    assert file1_5.is_file()
-    assert os.path.isfile(file2_5) is True
+    assert file2_5.is_file()
 
     # clean files freshly downloaded
     _clean_test_file(file1_5)
@@ -288,8 +304,8 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog,
 
     exchange = get_patched_exchange(mocker, default_conf)
 
-    file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
+    file1_1 = testdatadir / 'MEME_BTC-1m.json'
-    file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json')
+    file1_5 = testdatadir / 'MEME_BTC-5m.json'
     _backup_file(file1_1)
     _backup_file(file1_5)
 
@@ -359,43 +375,12 @@ def test_init(default_conf, mocker) -> None:
     )
 
 
-def test_trim_tickerlist() -> None:
+def test_trim_tickerlist(testdatadir) -> None:
-    file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json')
+    file = testdatadir / 'UNITTEST_BTC-1m.json'
     with open(file) as data_file:
         ticker_list = json.load(data_file)
     ticker_list_len = len(ticker_list)
 
-    # Test the pattern ^(-\d+)$
-    # This pattern uses the latest N elements
-    timerange = TimeRange(None, 'line', 0, -5)
-    ticker = trim_tickerlist(ticker_list, timerange)
-    ticker_len = len(ticker)
-
-    assert ticker_len == 5
-    assert ticker_list[0] is not ticker[0]  # The first element should be different
-    assert ticker_list[-1] is ticker[-1]  # The last element must be the same
-
-    # Test the pattern ^(\d+)-$
-    # This pattern keep X element from the end
-    timerange = TimeRange('line', None, 5, 0)
-    ticker = trim_tickerlist(ticker_list, timerange)
-    ticker_len = len(ticker)
-
-    assert ticker_len == 5
-    assert ticker_list[0] is ticker[0]  # The first element must be the same
-    assert ticker_list[-1] is not ticker[-1]  # The last element should be different
-
-    # Test the pattern ^(\d+)-(\d+)$
-    # This pattern extract a window
-    timerange = TimeRange('index', 'index', 5, 10)
-    ticker = trim_tickerlist(ticker_list, timerange)
-    ticker_len = len(ticker)
-
-    assert ticker_len == 5
-    assert ticker_list[0] is not ticker[0]  # The first element should be different
-    assert ticker_list[5] is ticker[0]  # The list starts at the index 5
-    assert ticker_list[9] is ticker[-1]  # The list ends at the index 9 (5 elements)
-
     # Test the pattern ^(\d{8})-(\d{8})$
     # This pattern extract a window between the dates
     timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1)
@@ -435,13 +420,6 @@ def test_trim_tickerlist() -> None:
 
     assert ticker_list_len == ticker_len
 
-    # Test invalid timerange (start after stop)
-    timerange = TimeRange('index', 'index', 10, 5)
-    with pytest.raises(ValueError, match=r'The timerange .* is incorrect'):
-        trim_tickerlist(ticker_list, timerange)
-
-    assert ticker_list_len == ticker_len
-
     # passing empty list
     timerange = TimeRange(None, None, None, 5)
     ticker = trim_tickerlist([], timerange)
@@ -449,22 +427,21 @@ def test_trim_tickerlist() -> None:
     assert not ticker
 
 
-def test_file_dump_json_tofile() -> None:
+def test_file_dump_json_tofile(testdatadir) -> None:
-    file = os.path.join(os.path.dirname(__file__), '..', 'testdata',
+    file = testdatadir / 'test_{id}.json'.format(id=str(uuid.uuid4()))
-                        'test_{id}.json'.format(id=str(uuid.uuid4())))
     data = {'bar': 'foo'}
 
     # check the file we will create does not exist
-    assert os.path.isfile(file) is False
+    assert not file.is_file()
 
     # Create the Json file
     file_dump_json(file, data)
 
     # Check the file was create
-    assert os.path.isfile(file) is True
+    assert file.is_file()
 
     # Open the Json file created and test the data is in it
-    with open(file) as data_file:
+    with file.open() as data_file:
         json_from_file = json.load(data_file)
 
     assert 'bar' in json_from_file
@@ -571,3 +548,92 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
     assert "ETH/BTC" in unav_pairs
     assert "XRP/BTC" in unav_pairs
     assert log_has("Skipping pair ETH/BTC...", caplog)
+
+
+def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
+    dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock())
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
+    )
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    mocker.patch.object(Path, "unlink", MagicMock())
+
+    ex = get_patched_exchange(mocker, default_conf)
+    timerange = TimeRange.parse_timerange("20190101-20190102")
+    unavailable_pairs = refresh_backtest_trades_data(exchange=ex,
+                                                     pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"],
+                                                     datadir=testdatadir,
+                                                     timerange=timerange, erase=True
+                                                     )
+
+    assert dl_mock.call_count == 2
+    assert dl_mock.call_args[1]['timerange'].starttype == 'date'
+
+    assert log_has("Downloading trades for pair ETH/BTC.", caplog)
+    assert unavailable_pairs == ["XRP/ETH"]
+    assert log_has("Skipping pair XRP/ETH...", caplog)
+
+
+def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog) -> None:
+
+    ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history))
+    mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
+                 ght_mock)
+    exchange = get_patched_exchange(mocker, default_conf)
+    file1 = testdatadir / 'ETH_BTC-trades.json.gz'
+
+    _backup_file(file1)
+
+    assert not file1.is_file()
+
+    assert download_trades_history(datadir=testdatadir, exchange=exchange,
+                                   pair='ETH/BTC')
+    assert log_has("New Amount of trades: 5", caplog)
+    assert file1.is_file()
+
+    # clean files freshly downloaded
+    _clean_test_file(file1)
+
+    mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
+                 MagicMock(side_effect=ValueError))
+
+    assert not download_trades_history(datadir=testdatadir, exchange=exchange,
+                                       pair='ETH/BTC')
+    assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)
+
+
+def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog):
+
+    pair = 'XRP/ETH'
+    file1 = testdatadir / 'XRP_ETH-1m.json'
+    file5 = testdatadir / 'XRP_ETH-5m.json'
+    # Compare downloaded dataset with converted dataset
+    dfbak_1m = history.load_pair_history(datadir=testdatadir,
+                                         ticker_interval="1m",
+                                         pair=pair)
+    dfbak_5m = history.load_pair_history(datadir=testdatadir,
+                                         ticker_interval="5m",
+                                         pair=pair)
+
+    _backup_file(file1, copy_file=True)
+    _backup_file(file5)
+
+    tr = TimeRange.parse_timerange('20191011-20191012')
+
+    convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'],
+                            datadir=testdatadir, timerange=tr, erase=True)
+
+    assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog)
+    # Load new data
+    df_1m = history.load_pair_history(datadir=testdatadir,
+                                      ticker_interval="1m",
+                                      pair=pair)
+    df_5m = history.load_pair_history(datadir=testdatadir,
+                                      ticker_interval="5m",
+                                      pair=pair)
+
+    assert df_1m.equals(dfbak_1m)
+    assert df_5m.equals(dfbak_5m)
+
+    _clean_test_file(file1)
+    _clean_test_file(file5)

tests/edge/test_edge.py

@@ -79,7 +79,7 @@ tc0 = BTContainer(data=[
     # D  O     H     L     C     V    B  S
     [0, 5000, 5025, 4975, 4987, 6172, 1, 0],
     [1, 5000, 5025, 4975, 4987, 6172, 0, 1]],  # enter trade (signal on last candle)
-    stop_loss=-0.99, roi=float('inf'), profit_perc=0.00,
+    stop_loss=-0.99, roi={"0": float('inf')}, profit_perc=0.00,
     trades=[]
 )
 
@@ -94,7 +94,7 @@ tc1 = BTContainer(data=[
     [5, 5000, 5025, 4975, 4987, 6172, 0, 1],  # no action
     [6, 5000, 5025, 4975, 4987, 6172, 0, 0],  # should sell
 ],
-    stop_loss=-0.99, roi=float('inf'), profit_perc=0.00,
+    stop_loss=-0.99, roi={"0": float('inf')}, profit_perc=0.00,
     trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=2),
             BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=4, close_tick=6)]
 )
@@ -106,7 +106,7 @@ tc2 = BTContainer(data=[
     [1, 5000, 5025, 4600, 4987, 6172, 0, 0],  # enter trade, stoploss hit
     [2, 5000, 5025, 4975, 4987, 6172, 0, 0],
 ],
-    stop_loss=-0.01, roi=float('inf'), profit_perc=-0.01,
+    stop_loss=-0.01, roi={"0": float('inf')}, profit_perc=-0.01,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)]
 )
 
@@ -117,7 +117,7 @@ tc3 = BTContainer(data=[
     [1, 5000, 5025, 4800, 4987, 6172, 0, 0],  # enter trade, stoploss hit
     [2, 5000, 5025, 4975, 4987, 6172, 0, 0],
 ],
-    stop_loss=-0.03, roi=float('inf'), profit_perc=-0.03,
+    stop_loss=-0.03, roi={"0": float('inf')}, profit_perc=-0.03,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)]
 )
 
@@ -128,7 +128,7 @@ tc4 = BTContainer(data=[
     [1, 5000, 5025, 4800, 4987, 6172, 0, 1],  # enter trade, stoploss hit, sell signal
     [2, 5000, 5025, 4975, 4987, 6172, 0, 0],
 ],
-    stop_loss=-0.03, roi=float('inf'), profit_perc=-0.03,
+    stop_loss=-0.03, roi={"0": float('inf')}, profit_perc=-0.03,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)]
 )
 

tests/exchange/test_exchange.py

@@ -18,7 +18,9 @@ from freqtrade.exchange.exchange import (API_RETRY_COUNT, timeframe_to_minutes,
                                          timeframe_to_msecs,
                                          timeframe_to_next_date,
                                          timeframe_to_prev_date,
-                                         timeframe_to_seconds)
+                                         timeframe_to_seconds,
+                                         symbol_is_pair,
+                                         market_is_active)
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
 from tests.conftest import get_patched_exchange, log_has, log_has_re
 
@@ -142,6 +144,12 @@ def test_exchange_resolver(default_conf, mocker, caplog):
     assert not log_has_re(r"No .* specific subclass found. Using the generic class instead.",
                           caplog)
 
+    # Test mapping
+    exchange = ExchangeResolver('binanceus', default_conf).exchange
+    assert isinstance(exchange, Exchange)
+    assert isinstance(exchange, Binance)
+    assert not isinstance(exchange, Kraken)
+
 
 def test_validate_order_time_in_force(default_conf, mocker, caplog):
     caplog.set_level(logging.INFO)
@@ -409,7 +417,8 @@ def test_validate_timeframes_failed(default_conf, mocker):
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={}))
     mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock())
-    with pytest.raises(OperationalException, match=r'Invalid ticker 3m, this Exchange supports.*'):
+    with pytest.raises(OperationalException,
+                       match=r"Invalid ticker interval '3m'. This exchange supports.*"):
         Exchange(default_conf)
 
 
@@ -922,17 +931,17 @@ def test_get_balances_prod(default_conf, mocker, exchange_name):
 def test_get_tickers(default_conf, mocker, exchange_name):
     api_mock = MagicMock()
     tick = {'ETH/BTC': {
-          'symbol': 'ETH/BTC',
+        'symbol': 'ETH/BTC',
-          'bid': 0.5,
+        'bid': 0.5,
-          'ask': 1,
+        'ask': 1,
-          'last': 42,
+        'last': 42,
-      }, 'BCH/BTC': {
+    }, 'BCH/BTC': {
-          'symbol': 'BCH/BTC',
+        'symbol': 'BCH/BTC',
-          'bid': 0.6,
+        'bid': 0.6,
-          'ask': 0.5,
+        'ask': 0.5,
-          'last': 41,
+        'last': 41,
-      }
+    }
-      }
+    }
     api_mock.fetch_tickers = MagicMock(return_value=tick)
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
     # retrieve original ticker
@@ -1135,6 +1144,13 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_
         await exchange._async_get_candle_history(pair, "5m",
                                                  (arrow.utcnow().timestamp - 2000) * 1000)
 
+    with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching '
+                                                   r'historical candlestick data\..*'):
+        api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        await exchange._async_get_candle_history(pair, "5m",
+                                                 (arrow.utcnow().timestamp - 2000) * 1000)
+
 
 @pytest.mark.asyncio
 async def test__async_get_candle_history_empty(default_conf, mocker, caplog):
@@ -1306,6 +1322,196 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na
     assert ticks[9][5] == 2.31452783
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name,
+                                   trades_history):
+
+    caplog.set_level(logging.DEBUG)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    # Monkey-patch async function
+    exchange._api_async.fetch_trades = get_mock_coro(trades_history)
+
+    pair = 'ETH/BTC'
+    res = await exchange._async_fetch_trades(pair, since=None, params=None)
+    assert type(res) is list
+    assert isinstance(res[0], dict)
+    assert isinstance(res[1], dict)
+
+    assert exchange._api_async.fetch_trades.call_count == 1
+    assert exchange._api_async.fetch_trades.call_args[0][0] == pair
+    assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000
+
+    assert log_has_re(f"Fetching trades for pair {pair}, since .*", caplog)
+    caplog.clear()
+    exchange._api_async.fetch_trades.reset_mock()
+    res = await exchange._async_fetch_trades(pair, since=None, params={'from': '123'})
+    assert exchange._api_async.fetch_trades.call_count == 1
+    assert exchange._api_async.fetch_trades.call_args[0][0] == pair
+    assert exchange._api_async.fetch_trades.call_args[1]['limit'] == 1000
+    assert exchange._api_async.fetch_trades.call_args[1]['params'] == {'from': '123'}
+    assert log_has_re(f"Fetching trades for pair {pair}, params: .*", caplog)
+
+    exchange = Exchange(default_conf)
+    await async_ccxt_exception(mocker, default_conf, MagicMock(),
+                               "_async_fetch_trades", "fetch_trades",
+                               pair='ABCD/BTC', since=None)
+
+    api_mock = MagicMock()
+    with pytest.raises(OperationalException, match=r'Could not fetch trade data*'):
+        api_mock.fetch_trades = MagicMock(side_effect=ccxt.BaseError("Unknown error"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000)
+
+    with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching '
+                                                   r'historical trade data\..*'):
+        api_mock.fetch_trades = MagicMock(side_effect=ccxt.NotSupported("Not supported"))
+        exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+        await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchange_name,
+                                           trades_history):
+
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    pagination_arg = exchange._trades_pagination_arg
+
+    async def mock_get_trade_hist(pair, *args, **kwargs):
+        if 'since' in kwargs:
+            # Return first 3
+            return trades_history[:-2]
+        elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3]['id']:
+            # Return 2
+            return trades_history[-3:-1]
+        else:
+            # Return last 2
+            return trades_history[-2:]
+    # Monkey-patch async function
+    exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
+
+    pair = 'ETH/BTC'
+    ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0]["timestamp"],
+                                                     until=trades_history[-1]["timestamp"]-1)
+    assert type(ret) is tuple
+    assert ret[0] == pair
+    assert type(ret[1]) is list
+    assert len(ret[1]) == len(trades_history)
+    assert exchange._async_fetch_trades.call_count == 3
+    fetch_trades_cal = exchange._async_fetch_trades.call_args_list
+    # first call (using since, not fromId)
+    assert fetch_trades_cal[0][0][0] == pair
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+
+    # 2nd call
+    assert fetch_trades_cal[1][0][0] == pair
+    assert 'params' in fetch_trades_cal[1][1]
+    assert exchange._ft_has['trades_pagination_arg'] in fetch_trades_cal[1][1]['params']
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+async def test__async_get_trade_history_time(default_conf, mocker, caplog, exchange_name,
+                                             trades_history):
+
+    caplog.set_level(logging.DEBUG)
+
+    async def mock_get_trade_hist(pair, *args, **kwargs):
+        if kwargs['since'] == trades_history[0]["timestamp"]:
+            return trades_history[:-1]
+        else:
+            return trades_history[-1:]
+
+    caplog.set_level(logging.DEBUG)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    # Monkey-patch async function
+    exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
+    pair = 'ETH/BTC'
+    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
+                                                       until=trades_history[-1]["timestamp"]-1)
+    assert type(ret) is tuple
+    assert ret[0] == pair
+    assert type(ret[1]) is list
+    assert len(ret[1]) == len(trades_history)
+    assert exchange._async_fetch_trades.call_count == 2
+    fetch_trades_cal = exchange._async_fetch_trades.call_args_list
+    # first call (using since, not fromId)
+    assert fetch_trades_cal[0][0][0] == pair
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+
+    # 2nd call
+    assert fetch_trades_cal[1][0][0] == pair
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert log_has_re(r"Stopping because until was reached.*", caplog)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, exchange_name,
+                                                   trades_history):
+
+    caplog.set_level(logging.DEBUG)
+
+    async def mock_get_trade_hist(pair, *args, **kwargs):
+        if kwargs['since'] == trades_history[0]["timestamp"]:
+            return trades_history[:-1]
+        else:
+            return []
+
+    caplog.set_level(logging.DEBUG)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    # Monkey-patch async function
+    exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
+    pair = 'ETH/BTC'
+    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
+                                                       until=trades_history[-1]["timestamp"]-1)
+    assert type(ret) is tuple
+    assert ret[0] == pair
+    assert type(ret[1]) is list
+    assert len(ret[1]) == len(trades_history) - 1
+    assert exchange._async_fetch_trades.call_count == 2
+    fetch_trades_cal = exchange._async_fetch_trades.call_args_list
+    # first call (using since, not fromId)
+    assert fetch_trades_cal[0][0][0] == pair
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+
+
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_get_historic_trades(default_conf, mocker, caplog, exchange_name, trades_history):
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+
+    pair = 'ETH/BTC'
+
+    exchange._async_get_trade_history_id = get_mock_coro((pair, trades_history))
+    exchange._async_get_trade_history_time = get_mock_coro((pair, trades_history))
+    ret = exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
+                                       until=trades_history[-1]["timestamp"])
+
+    # Depending on the exchange, one or the other method should be called
+    assert sum([exchange._async_get_trade_history_id.call_count,
+                exchange._async_get_trade_history_time.call_count]) == 1
+
+    assert len(ret) == 2
+    assert ret[0] == pair
+    assert len(ret[1]) == len(trades_history)
+
+
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange_name,
+                                          trades_history):
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=False)
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+
+    pair = 'ETH/BTC'
+
+    with pytest.raises(OperationalException,
+                       match="This exchange does not suport downloading Trades."):
+        exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
+                                     until=trades_history[-1]["timestamp"])
+
+
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
     default_conf['dry_run'] = True
@@ -1452,13 +1658,17 @@ def test_merge_ft_has_dict(default_conf, mocker):
     assert ex._ft_has == Exchange._ft_has_default
 
     ex = Kraken(default_conf)
-    assert ex._ft_has == Exchange._ft_has_default
+    assert ex._ft_has != Exchange._ft_has_default
+    assert ex._ft_has['trades_pagination'] == 'id'
+    assert ex._ft_has['trades_pagination_arg'] == 'since'
 
     # Binance defines different values
     ex = Binance(default_conf)
     assert ex._ft_has != Exchange._ft_has_default
     assert ex._ft_has['stoploss_on_exchange']
     assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc']
+    assert ex._ft_has['trades_pagination'] == 'id'
+    assert ex._ft_has['trades_pagination_arg'] == 'fromId'
 
     conf = copy.deepcopy(default_conf)
     conf['exchange']['_ft_has_params'] = {"DeadBeef": 20,
@@ -1485,6 +1695,74 @@ def test_get_valid_pair_combination(default_conf, mocker, markets):
         ex.get_valid_pair_combination("NOPAIR", "ETH")
 
 
+@pytest.mark.parametrize(
+    "base_currencies, quote_currencies, pairs_only, active_only, expected_keys", [
+        # Testing markets (in conftest.py):
+        # 'BLK/BTC':  'active': True
+        # 'BTT/BTC':  'active': True
+        # 'ETH/BTC':  'active': True
+        # 'ETH/USDT': 'active': True
+        # 'LTC/BTC':  'active': False
+        # 'LTC/USD':  'active': True
+        # 'LTC/USDT': 'active': True
+        # 'NEO/BTC':  'active': False
+        # 'TKN/BTC':  'active'  not set
+        # 'XLTCUSDT': 'active': True, not a pair
+        # 'XRP/BTC':  'active': False
+        # all markets
+        ([], [], False, False,
+         ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD',
+          'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']),
+        # active markets
+        ([], [], False, True,
+         ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/USD', 'LTC/USDT',
+          'TKN/BTC', 'XLTCUSDT']),
+        # all pairs
+        ([], [], True, False,
+         ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD',
+          'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']),
+        # active pairs
+        ([], [], True, True,
+         ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/USD', 'LTC/USDT', 'TKN/BTC']),
+        # all markets, base=ETH, LTC
+        (['ETH', 'LTC'], [], False, False,
+         ['ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']),
+        # all markets, base=LTC
+        (['LTC'], [], False, False,
+         ['LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']),
+        # all markets, quote=USDT
+        ([], ['USDT'], False, False,
+         ['ETH/USDT', 'LTC/USDT', 'XLTCUSDT']),
+        # all markets, quote=USDT, USD
+        ([], ['USDT', 'USD'], False, False,
+         ['ETH/USDT', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']),
+        # all markets, base=LTC, quote=USDT
+        (['LTC'], ['USDT'], False, False,
+         ['LTC/USDT', 'XLTCUSDT']),
+        # all pairs, base=LTC, quote=USDT
+        (['LTC'], ['USDT'], True, False,
+         ['LTC/USDT']),
+        # all markets, base=LTC, quote=USDT, NONEXISTENT
+        (['LTC'], ['USDT', 'NONEXISTENT'], False, False,
+         ['LTC/USDT', 'XLTCUSDT']),
+        # all markets, base=LTC, quote=NONEXISTENT
+        (['LTC'], ['NONEXISTENT'], False, False,
+         []),
+    ])
+def test_get_markets(default_conf, mocker, markets,
+                     base_currencies, quote_currencies, pairs_only, active_only,
+                     expected_keys):
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          _init_ccxt=MagicMock(return_value=MagicMock()),
+                          _load_async_markets=MagicMock(),
+                          validate_pairs=MagicMock(),
+                          validate_timeframes=MagicMock(),
+                          markets=PropertyMock(return_value=markets))
+    ex = Exchange(default_conf)
+    pairs = ex.get_markets(base_currencies, quote_currencies, pairs_only, active_only)
+    assert sorted(pairs.keys()) == sorted(expected_keys)
+
+
 def test_timeframe_to_minutes():
     assert timeframe_to_minutes("5m") == 5
     assert timeframe_to_minutes("10m") == 10
@@ -1554,3 +1832,33 @@ def test_timeframe_to_next_date():
 
     date = datetime.now(tz=timezone.utc)
     assert timeframe_to_next_date("5m") > date
+
+
+@pytest.mark.parametrize("market_symbol,base_currency,quote_currency,expected_result", [
+    ("BTC/USDT", None, None, True),
+    ("USDT/BTC", None, None, True),
+    ("BTCUSDT", None, None, False),
+    ("BTC/USDT", None, "USDT", True),
+    ("USDT/BTC", None, "USDT", False),
+    ("BTCUSDT", None, "USDT", False),
+    ("BTC/USDT", "BTC", None, True),
+    ("USDT/BTC", "BTC", None, False),
+    ("BTCUSDT", "BTC", None, False),
+    ("BTC/USDT", "BTC", "USDT", True),
+    ("BTC/USDT", "USDT", "BTC", False),
+    ("BTC/USDT", "BTC", "USD", False),
+    ("BTCUSDT", "BTC", "USDT", False),
+    ("BTC/", None, None, False),
+    ("/USDT", None, None, False),
+])
+def test_symbol_is_pair(market_symbol, base_currency, quote_currency, expected_result) -> None:
+    assert symbol_is_pair(market_symbol, base_currency, quote_currency) == expected_result
+
+
+@pytest.mark.parametrize("market,expected_result", [
+    ({'symbol': 'ETH/BTC', 'active': True}, True),
+    ({'symbol': 'ETH/BTC', 'active': False}, False),
+    ({'symbol': 'ETH/BTC', }, True),
+])
+def test_market_is_active(market, expected_result) -> None:
+    assert market_is_active(market) == expected_result

tests/optimize/__init__.py

@@ -1,4 +1,4 @@
-from typing import NamedTuple, List
+from typing import Dict, List, NamedTuple
 
 import arrow
 from pandas import DataFrame
@@ -25,7 +25,7 @@ class BTContainer(NamedTuple):
     """
     data: List[float]
     stop_loss: float
-    roi: float
+    roi: Dict[str, float]
     trades: List[BTrade]
     profit_perc: float
     trailing_stop: bool = False

tests/optimize/test_backtest_detail.py

@@ -22,7 +22,7 @@ tc0 = BTContainer(data=[
     [3, 5010, 5000, 4980, 5010, 6172, 0, 1],
     [4, 5010, 4987, 4977, 4995, 6172, 0, 0],
     [5, 4995, 4995, 4995, 4950, 6172, 0, 0]],
-    stop_loss=-0.01, roi=1, profit_perc=0.002, use_sell_signal=True,
+    stop_loss=-0.01, roi={"0": 1}, profit_perc=0.002, use_sell_signal=True,
     trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=4)]
 )
 
@@ -36,7 +36,7 @@ tc1 = BTContainer(data=[
     [3, 4975, 5000, 4980, 4977, 6172, 0, 0],
     [4, 4977, 4987, 4977, 4995, 6172, 0, 0],
     [5, 4995, 4995, 4995, 4950, 6172, 0, 0]],
-    stop_loss=-0.01, roi=1, profit_perc=-0.01,
+    stop_loss=-0.01, roi={"0": 1}, profit_perc=-0.01,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)]
 )
 
@@ -51,7 +51,7 @@ tc2 = BTContainer(data=[
     [3, 4975, 5000, 4800, 4962, 6172, 0, 0],  # exit with stoploss hit
     [4, 4962, 4987, 4937, 4950, 6172, 0, 0],
     [5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
-    stop_loss=-0.03, roi=1, profit_perc=-0.03,
+    stop_loss=-0.03, roi={"0": 1}, profit_perc=-0.03,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=3)]
 )
 
@@ -71,7 +71,7 @@ tc3 = BTContainer(data=[
     [4, 4975, 5000, 4950, 4962, 6172, 0, 0],  # enter trade 2 (signal on last candle)
     [5, 4962, 4987, 4000, 4000, 6172, 0, 0],  # exit with stoploss hit
     [6, 4950, 4975, 4975, 4950, 6172, 0, 0]],
-    stop_loss=-0.02, roi=1, profit_perc=-0.04,
+    stop_loss=-0.02, roi={"0": 1}, profit_perc=-0.04,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2),
             BTrade(sell_reason=SellType.STOP_LOSS, open_tick=4, close_tick=5)]
 )
@@ -88,7 +88,7 @@ tc4 = BTContainer(data=[
     [3, 4975, 5000, 4950, 4962, 6172, 0, 0],
     [4, 4962, 4987, 4937, 4950, 6172, 0, 0],
     [5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
-    stop_loss=-0.02, roi=0.06, profit_perc=-0.02,
+    stop_loss=-0.02, roi={"0": 0.06}, profit_perc=-0.02,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)]
 )
 
@@ -102,7 +102,7 @@ tc5 = BTContainer(data=[
     [3, 4975, 6000, 4975, 6000, 6172, 0, 0],  # ROI
     [4, 4962, 4987, 4972, 4950, 6172, 0, 0],
     [5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
-    stop_loss=-0.01, roi=0.03, profit_perc=0.03,
+    stop_loss=-0.01, roi={"0": 0.03}, profit_perc=0.03,
     trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)]
 )
 
@@ -116,7 +116,7 @@ tc6 = BTContainer(data=[
     [3, 4975, 5000, 4950, 4962, 6172, 0, 0],
     [4, 4962, 4987, 4972, 4950, 6172, 0, 0],
     [5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
-    stop_loss=-0.02, roi=0.05, profit_perc=-0.02,
+    stop_loss=-0.02, roi={"0": 0.05}, profit_perc=-0.02,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)]
 )
 
@@ -130,7 +130,7 @@ tc7 = BTContainer(data=[
     [3, 4975, 5000, 4950, 4962, 6172, 0, 0],
     [4, 4962, 4987, 4972, 4950, 6172, 0, 0],
     [5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
-    stop_loss=-0.02, roi=0.03, profit_perc=0.03,
+    stop_loss=-0.02, roi={"0": 0.03}, profit_perc=0.03,
     trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)]
 )
 
@@ -144,7 +144,7 @@ tc8 = BTContainer(data=[
     [2, 5000, 5250, 4750, 4850, 6172, 0, 0],
     [3, 4850, 5050, 4650, 4750, 6172, 0, 0],
     [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
-    stop_loss=-0.10, roi=0.10, profit_perc=-0.055, trailing_stop=True,
+    stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.055, trailing_stop=True,
     trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)]
 )
 
@@ -158,7 +158,7 @@ tc9 = BTContainer(data=[
     [2, 5000, 5050, 4950, 5000, 6172, 0, 0],
     [3, 5000, 5200, 4550, 4850, 6172, 0, 0],
     [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
-    stop_loss=-0.10, roi=0.10, profit_perc=-0.064, trailing_stop=True,
+    stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.064, trailing_stop=True,
     trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)]
 )
 
@@ -172,7 +172,7 @@ tc10 = BTContainer(data=[
     [2, 5100, 5251, 5100, 5100, 6172, 0, 0],
     [3, 4850, 5050, 4650, 4750, 6172, 0, 0],
     [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
-    stop_loss=-0.10, roi=0.10, profit_perc=-0.1, trailing_stop=True,
+    stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.1, trailing_stop=True,
     trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.10,
     trailing_stop_positive=0.03,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=4)]
@@ -188,7 +188,7 @@ tc11 = BTContainer(data=[
     [2, 5100, 5251, 5100, 5100, 6172, 0, 0],
     [3, 4850, 5050, 4650, 4750, 6172, 0, 0],
     [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
-    stop_loss=-0.10, roi=0.10, profit_perc=0.019, trailing_stop=True,
+    stop_loss=-0.10, roi={"0": 0.10}, profit_perc=0.019, trailing_stop=True,
     trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05,
     trailing_stop_positive=0.03,
     trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)]
@@ -204,7 +204,7 @@ tc12 = BTContainer(data=[
     [2, 5100, 5251, 4650, 5100, 6172, 0, 0],
     [3, 4850, 5050, 4650, 4750, 6172, 0, 0],
     [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
-    stop_loss=-0.10, roi=0.10, profit_perc=0.019, trailing_stop=True,
+    stop_loss=-0.10, roi={"0": 0.10}, profit_perc=0.019, trailing_stop=True,
     trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05,
     trailing_stop_positive=0.03,
     trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=2)]
@@ -219,7 +219,7 @@ tc13 = BTContainer(data=[
     [2, 5100, 5251, 4850, 5100, 6172, 0, 0],
     [3, 4850, 5050, 4850, 4750, 6172, 0, 0],
     [4, 4750, 4950, 4850, 4750, 6172, 0, 0]],
-    stop_loss=-0.10, roi=0.01, profit_perc=0.01,
+    stop_loss=-0.10, roi={"0": 0.01}, profit_perc=0.01,
     trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=1)]
 )
 
@@ -232,7 +232,7 @@ tc14 = BTContainer(data=[
     [2, 5100, 5251, 4850, 5100, 6172, 0, 0],
     [3, 4850, 5050, 4850, 4750, 6172, 0, 0],
     [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
-    stop_loss=-0.05, roi=0.10, profit_perc=-0.05,
+    stop_loss=-0.05, roi={"0": 0.10}, profit_perc=-0.05,
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)]
 )
 
@@ -246,11 +246,26 @@ tc15 = BTContainer(data=[
     [2, 5100, 5251, 4650, 5100, 6172, 0, 0],
     [3, 4850, 5050, 4850, 4750, 6172, 0, 0],
     [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
-    stop_loss=-0.05, roi=0.01, profit_perc=-0.04,
+    stop_loss=-0.05, roi={"0": 0.01}, profit_perc=-0.04,
     trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=1),
             BTrade(sell_reason=SellType.STOP_LOSS, open_tick=2, close_tick=2)]
 )
 
+# Test 16: Buy, hold for 65 mins, then forcesell using roi=-1
+# Causes negative profit even though sell-reason is ROI.
+# stop-loss: 10%, ROI: 10% (should not apply), -100% after 65 minutes (limits trade duration)
+tc16 = BTContainer(data=[
+    # D  O     H     L     C     V    B  S
+    [0, 5000, 5025, 4975, 4987, 6172, 1, 0],
+    [1, 5000, 5025, 4975, 4987, 6172, 0, 0],
+    [2, 4987, 5300, 4950, 5050, 6172, 0, 0],
+    [3, 4975, 5000, 4940, 4962, 6172, 0, 0],  # ForceSell on ROI (roi=-1)
+    [4, 4962, 4987, 4972, 4950, 6172, 0, 0],
+    [5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
+    stop_loss=-0.10, roi={"0": 0.10, "65": -1}, profit_perc=-0.012,
+    trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)]
+)
+
 TESTS = [
     tc0,
     tc1,
@@ -268,6 +283,7 @@ TESTS = [
     tc13,
     tc14,
     tc15,
+    tc16,
 ]
 
 
@@ -277,7 +293,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
     run functional tests
     """
     default_conf["stoploss"] = data.stop_loss
-    default_conf["minimal_roi"] = {"0": data.roi}
+    default_conf["minimal_roi"] = data.roi
     default_conf["ticker_interval"] = tests_ticker_interval
     default_conf["trailing_stop"] = data.trailing_stop
     default_conf["trailing_only_offset_is_reached"] = data.trailing_only_offset_is_reached
@@ -285,7 +301,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
     if data.trailing_stop_positive:
         default_conf["trailing_stop_positive"] = data.trailing_stop_positive
     default_conf["trailing_stop_positive_offset"] = data.trailing_stop_positive_offset
-    default_conf["experimental"] = {"use_sell_signal": data.use_sell_signal}
+    default_conf["ask_strategy"] = {"use_sell_signal": data.use_sell_signal}
 
     mocker.patch("freqtrade.exchange.Exchange.get_fee", MagicMock(return_value=0.0))
     patch_exchange(mocker)

tests/optimize/test_backtesting.py

@@ -26,6 +26,21 @@ from tests.conftest import (get_args, log_has, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
 
 
+ORDER_TYPES = [
+    {
+        'buy': 'limit',
+        'sell': 'limit',
+        'stoploss': 'limit',
+        'stoploss_on_exchange': False
+    },
+    {
+        'buy': 'limit',
+        'sell': 'limit',
+        'stoploss': 'limit',
+        'stoploss_on_exchange': True
+    }]
+
+
 def trim_dictlist(dict_list, num):
     new = {}
     for pair, pair_data in dict_list.items():
@@ -34,7 +49,7 @@ def trim_dictlist(dict_list, num):
 
 
 def load_data_test(what, testdatadir):
-    timerange = TimeRange(None, 'line', 0, -101)
+    timerange = TimeRange.parse_timerange('1510694220-1510700340')
     pair = history.load_tickerdata_file(testdatadir, ticker_interval='1m',
                                         pair='UNITTEST/BTC', timerange=timerange)
     datalen = len(pair)
@@ -211,7 +226,8 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
         '--disable-max-market-positions',
         '--timerange', ':100',
         '--export', '/bar/foo',
-        '--export-filename', 'foo_bar.json'
+        '--export-filename', 'foo_bar.json',
+        '--fee', '0',
     ]
 
     config = setup_configuration(get_args(args), RunMode.BACKTEST)
@@ -243,6 +259,9 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
     assert 'exportfilename' in config
     assert log_has('Storing backtest results to {} ...'.format(config['exportfilename']), caplog)
 
+    assert 'fee' in config
+    assert log_has('Parameter --fee detected, setting fee to: {} ...'.format(config['fee']), caplog)
+
 
 def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None:
     default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
@@ -277,21 +296,6 @@ def test_start(mocker, fee, default_conf, caplog) -> None:
     assert start_mock.call_count == 1
 
 
-ORDER_TYPES = [
-    {
-        'buy': 'limit',
-        'sell': 'limit',
-        'stoploss': 'limit',
-        'stoploss_on_exchange': False
-    },
-    {
-        'buy': 'limit',
-        'sell': 'limit',
-        'stoploss': 'limit',
-        'stoploss_on_exchange': True
-    }]
-
-
 @pytest.mark.parametrize("order_types", ORDER_TYPES)
 def test_backtesting_init(mocker, default_conf, order_types) -> None:
     """
@@ -314,10 +318,6 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None:
 
 
 def test_backtesting_init_no_ticker_interval(mocker, default_conf, caplog) -> None:
-    """
-    Check that stoploss_on_exchange is set to False while backtesting
-    since backtesting assumes a perfect stoploss anyway.
-    """
     patch_exchange(mocker)
     del default_conf['ticker_interval']
     default_conf['strategy_list'] = ['DefaultStrategy',
@@ -330,9 +330,20 @@ def test_backtesting_init_no_ticker_interval(mocker, default_conf, caplog) -> No
             "or as cli argument `--ticker-interval 5m`", caplog)
 
 
+def test_tickerdata_with_fee(default_conf, mocker, testdatadir) -> None:
+    patch_exchange(mocker)
+    default_conf['fee'] = 0.1234
+
+    fee_mock = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5))
+    backtesting = Backtesting(default_conf)
+    assert backtesting.fee == 0.1234
+    assert fee_mock.call_count == 0
+
+
 def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None:
     patch_exchange(mocker)
-    timerange = TimeRange(None, 'line', 0, -100)
+    # timerange = TimeRange(None, 'line', 0, -100)
+    timerange = TimeRange.parse_timerange('1510694220-1510700340')
     tick = history.load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange)
     tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC",
                                                          fill_missing=True)}
@@ -464,7 +475,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
     default_conf['ticker_interval'] = '1m'
     default_conf['datadir'] = testdatadir
     default_conf['export'] = None
-    default_conf['timerange'] = '-100'
+    default_conf['timerange'] = '-1510694220'
 
     backtesting = Backtesting(default_conf)
     backtesting.start()
@@ -507,11 +518,12 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
 
 
 def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
+    default_conf['ask_strategy']['use_sell_signal'] = False
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
     patch_exchange(mocker)
     backtesting = Backtesting(default_conf)
     pair = 'UNITTEST/BTC'
-    timerange = TimeRange(None, 'line', 0, -201)
+    timerange = TimeRange('date', None, 1517227800, 0)
     data = history.load_data(datadir=testdatadir, ticker_interval='5m', pairs=['UNITTEST/BTC'],
                              timerange=timerange)
     data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
@@ -561,12 +573,13 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
 
 
 def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) -> None:
+    default_conf['ask_strategy']['use_sell_signal'] = False
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
     patch_exchange(mocker)
     backtesting = Backtesting(default_conf)
 
     # Run a backtesting for an exiting 1min ticker_interval
-    timerange = TimeRange(None, 'line', 0, -200)
+    timerange = TimeRange.parse_timerange('1510688220-1510700340')
     data = history.load_data(datadir=testdatadir, ticker_interval='1m', pairs=['UNITTEST/BTC'],
                              timerange=timerange)
     processed = backtesting.strategy.tickerdata_to_dataframe(data)
@@ -603,8 +616,6 @@ def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir) -> None:
     # TODO: Evaluate usefullness of this, the patterns and buy-signls are unrealistic
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
     tests = [['raise', 19], ['lower', 0], ['sine', 35]]
-    # We need to enable sell-signal - otherwise it sells on ROI!!
-    default_conf['experimental'] = {"use_sell_signal": True}
 
     for [contour, numres] in tests:
         simple_backtest(default_conf, contour, numres, mocker, testdatadir)
@@ -645,8 +656,6 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir):
     mocker.patch('freqtrade.optimize.backtesting.file_dump_json', MagicMock())
     backtest_conf = _make_backtest_conf(mocker, conf=default_conf,
                                         pair='UNITTEST/BTC', datadir=testdatadir)
-    # We need to enable sell-signal - otherwise it sells on ROI!!
-    default_conf['experimental'] = {"use_sell_signal": True}
     default_conf['ticker_interval'] = '1m'
     backtesting = Backtesting(default_conf)
     backtesting.strategy.advise_buy = _trend_alternate  # Override
@@ -687,8 +696,6 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
 
     # Remove data for one pair from the beginning of the data
     data[pair] = data[pair][tres:].reset_index()
-    # We need to enable sell-signal - otherwise it sells on ROI!!
-    default_conf['experimental'] = {"use_sell_signal": True}
     default_conf['ticker_interval'] = '5m'
 
     backtesting = Backtesting(default_conf)
@@ -817,7 +824,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
         '--datadir', str(testdatadir),
         'backtesting',
         '--ticker-interval', '1m',
-        '--timerange', '-100',
+        '--timerange', '1510694220-1510700340',
         '--enable-position-stacking',
         '--disable-max-market-positions'
     ]
@@ -827,7 +834,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
     exists = [
         'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
         'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
-        'Parameter --timerange detected: -100 ...',
+        'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',
@@ -863,7 +870,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
         '--datadir', str(testdatadir),
         'backtesting',
         '--ticker-interval', '1m',
-        '--timerange', '-100',
+        '--timerange', '1510694220-1510700340',
         '--enable-position-stacking',
         '--disable-max-market-positions',
         '--strategy-list',
@@ -881,7 +888,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
     exists = [
         'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
         'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
-        'Parameter --timerange detected: -100 ...',
+        'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Using stake_currency: BTC ...',
         'Using stake_amount: 0.001 ...',

tests/optimize/test_edge_cli.py

@@ -98,6 +98,16 @@ def test_edge_init(mocker, edge_conf) -> None:
     assert callable(edge_cli.edge.calculate)
 
 
+def test_edge_init_fee(mocker, edge_conf) -> None:
+    patch_exchange(mocker)
+    edge_conf['fee'] = 0.1234
+    edge_conf['stake_amount'] = 20
+    fee_mock = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5))
+    edge_cli = EdgeCli(edge_conf)
+    assert edge_cli.edge.fee == 0.1234
+    assert fee_mock.call_count == 0
+
+
 def test_generate_edge_table(edge_conf, mocker):
     patch_exchange(mocker)
     edge_cli = EdgeCli(edge_conf)

tests/optimize/test_hyperopt.py

@@ -12,7 +12,7 @@ from freqtrade import OperationalException
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.data.history import load_tickerdata_file
 from freqtrade.optimize import setup_configuration, start_hyperopt
-from freqtrade.optimize.default_hyperopt import DefaultHyperOpts
+from freqtrade.optimize.default_hyperopt import DefaultHyperOpt
 from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss
 from freqtrade.optimize.hyperopt import Hyperopt
 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver,
@@ -148,12 +148,12 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo
 def test_hyperoptresolver(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
 
-    hyperopts = DefaultHyperOpts
+    hyperopt = DefaultHyperOpt
-    delattr(hyperopts, 'populate_buy_trend')
+    delattr(hyperopt, 'populate_buy_trend')
-    delattr(hyperopts, 'populate_sell_trend')
+    delattr(hyperopt, 'populate_sell_trend')
     mocker.patch(
         'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver._load_hyperopt',
-        MagicMock(return_value=hyperopts(default_conf))
+        MagicMock(return_value=hyperopt(default_conf))
     )
     x = HyperOptResolver(default_conf, ).hyperopt
     assert not hasattr(x, 'populate_buy_trend')
@@ -516,7 +516,7 @@ def test_generate_optimizer(mocker, default_conf) -> None:
     default_conf.update({'hyperopt_min_trades': 1})
 
     trades = [
-        ('POWR/BTC', 0.023117, 0.000233, 100)
+        ('TRX/BTC', 0.023117, 0.000233, 100)
     ]
     labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration']
     backtest_result = pd.DataFrame.from_records(trades, columns=labels)

tests/pytest.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo "Running Unit tests"
+
+pytest --random-order --cov=freqtrade --cov-config=.coveragerc tests/

tests/strategy/legacy_strategy.py

@@ -6,8 +6,6 @@ from pandas import DataFrame
 
 # Add your lib to import here
 import talib.abstract as ta
-import freqtrade.vendor.qtpylib.indicators as qtpylib
-import numpy  # noqa
 
 
 # This class is a sample. Feel free to customize it.
@@ -17,7 +15,6 @@ class TestStrategyLegacy(IStrategy):
     removed in a future update.
     Please do not use this as a template, but refer to user_data/strategy/sample_strategy.py
     for a uptodate version of this template.
-
     """
 
     # Minimal ROI designed for the strategy.
@@ -51,156 +48,9 @@ class TestStrategyLegacy(IStrategy):
         # ADX
         dataframe['adx'] = ta.ADX(dataframe)
 
-        """
-        # Awesome oscillator
-        dataframe['ao'] = qtpylib.awesome_oscillator(dataframe)
-
-        # Commodity Channel Index: values Oversold:<-100, Overbought:>100
-        dataframe['cci'] = ta.CCI(dataframe)
-
-        # MACD
-        macd = ta.MACD(dataframe)
-        dataframe['macd'] = macd['macd']
-        dataframe['macdsignal'] = macd['macdsignal']
-        dataframe['macdhist'] = macd['macdhist']
-
-        # MFI
-        dataframe['mfi'] = ta.MFI(dataframe)
-
-        # Minus Directional Indicator / Movement
-        dataframe['minus_dm'] = ta.MINUS_DM(dataframe)
-        dataframe['minus_di'] = ta.MINUS_DI(dataframe)
-
-        # Plus Directional Indicator / Movement
-        dataframe['plus_dm'] = ta.PLUS_DM(dataframe)
-        dataframe['plus_di'] = ta.PLUS_DI(dataframe)
-        dataframe['minus_di'] = ta.MINUS_DI(dataframe)
-
-        # ROC
-        dataframe['roc'] = ta.ROC(dataframe)
-
-        # RSI
-        dataframe['rsi'] = ta.RSI(dataframe)
-
-        # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy)
-        rsi = 0.1 * (dataframe['rsi'] - 50)
-        dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1)
-
-        # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy)
-        dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1)
-
-        # Stoch
-        stoch = ta.STOCH(dataframe)
-        dataframe['slowd'] = stoch['slowd']
-        dataframe['slowk'] = stoch['slowk']
-
-        # Stoch fast
-        stoch_fast = ta.STOCHF(dataframe)
-        dataframe['fastd'] = stoch_fast['fastd']
-        dataframe['fastk'] = stoch_fast['fastk']
-
-        # Stoch RSI
-        stoch_rsi = ta.STOCHRSI(dataframe)
-        dataframe['fastd_rsi'] = stoch_rsi['fastd']
-        dataframe['fastk_rsi'] = stoch_rsi['fastk']
-        """
-
-        # Overlap Studies
-        # ------------------------------------
-
-        # Bollinger bands
-        bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
-        dataframe['bb_lowerband'] = bollinger['lower']
-        dataframe['bb_middleband'] = bollinger['mid']
-        dataframe['bb_upperband'] = bollinger['upper']
-
-        """
-        # EMA - Exponential Moving Average
-        dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3)
-        dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5)
-        dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
-        dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50)
-        dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
-
-        # SAR Parabol
-        dataframe['sar'] = ta.SAR(dataframe)
-
-        # SMA - Simple Moving Average
-        dataframe['sma'] = ta.SMA(dataframe, timeperiod=40)
-        """
-
         # TEMA - Triple Exponential Moving Average
         dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9)
 
-        # Cycle Indicator
-        # ------------------------------------
-        # Hilbert Transform Indicator - SineWave
-        hilbert = ta.HT_SINE(dataframe)
-        dataframe['htsine'] = hilbert['sine']
-        dataframe['htleadsine'] = hilbert['leadsine']
-
-        # Pattern Recognition - Bullish candlestick patterns
-        # ------------------------------------
-        """
-        # Hammer: values [0, 100]
-        dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe)
-        # Inverted Hammer: values [0, 100]
-        dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe)
-        # Dragonfly Doji: values [0, 100]
-        dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe)
-        # Piercing Line: values [0, 100]
-        dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100]
-        # Morningstar: values [0, 100]
-        dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100]
-        # Three White Soldiers: values [0, 100]
-        dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100]
-        """
-
-        # Pattern Recognition - Bearish candlestick patterns
-        # ------------------------------------
-        """
-        # Hanging Man: values [0, 100]
-        dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe)
-        # Shooting Star: values [0, 100]
-        dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe)
-        # Gravestone Doji: values [0, 100]
-        dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe)
-        # Dark Cloud Cover: values [0, 100]
-        dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe)
-        # Evening Doji Star: values [0, 100]
-        dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe)
-        # Evening Star: values [0, 100]
-        dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe)
-        """
-
-        # Pattern Recognition - Bullish/Bearish candlestick patterns
-        # ------------------------------------
-        """
-        # Three Line Strike: values [0, -100, 100]
-        dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe)
-        # Spinning Top: values [0, -100, 100]
-        dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100]
-        # Engulfing: values [0, -100, 100]
-        dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100]
-        # Harami: values [0, -100, 100]
-        dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100]
-        # Three Outside Up/Down: values [0, -100, 100]
-        dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100]
-        # Three Inside Up/Down: values [0, -100, 100]
-        dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100]
-        """
-
-        # Chart type
-        # ------------------------------------
-        """
-        # Heikinashi stategy
-        heikinashi = qtpylib.heikinashi(dataframe)
-        dataframe['ha_open'] = heikinashi['open']
-        dataframe['ha_close'] = heikinashi['close']
-        dataframe['ha_high'] = heikinashi['high']
-        dataframe['ha_low'] = heikinashi['low']
-        """
-
         return dataframe
 
     def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
@@ -212,8 +62,8 @@ class TestStrategyLegacy(IStrategy):
         dataframe.loc[
             (
                 (dataframe['adx'] > 30) &
-                (dataframe['tema'] <= dataframe['bb_middleband']) &
+                (dataframe['tema'] > dataframe['tema'].shift(1)) &
-                (dataframe['tema'] > dataframe['tema'].shift(1))
+                (dataframe['volume'] > 0)
             ),
             'buy'] = 1
 
@@ -228,8 +78,8 @@ class TestStrategyLegacy(IStrategy):
         dataframe.loc[
             (
                 (dataframe['adx'] > 70) &
-                (dataframe['tema'] > dataframe['bb_middleband']) &
+                (dataframe['tema'] < dataframe['tema'].shift(1)) &
-                (dataframe['tema'] < dataframe['tema'].shift(1))
+                (dataframe['volume'] > 0)
             ),
             'sell'] = 1
         return dataframe

tests/strategy/test_interface.py

@@ -106,7 +106,7 @@ def test_get_signal_handles_exceptions(mocker, default_conf):
 def test_tickerdata_to_dataframe(default_conf, testdatadir) -> None:
     strategy = DefaultStrategy(default_conf)
 
-    timerange = TimeRange(None, 'line', 0, -100)
+    timerange = TimeRange.parse_timerange('1510694220-1510700340')
     tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange)
     tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC",
                                                          fill_missing=True)}

tests/strategy/test_strategy.py

@@ -1,6 +1,5 @@
 # pragma pylint: disable=missing-docstring, protected-access, C0103
 import logging
-import tempfile
 import warnings
 from base64 import urlsafe_b64encode
 from os import path
@@ -39,7 +38,7 @@ def test_search_strategy():
 def test_load_strategy(default_conf, result):
     default_conf.update({'strategy': 'SampleStrategy'})
     resolver = StrategyResolver(default_conf)
-    assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
+    assert 'rsi' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
 
 
 def test_load_strategy_base64(result, caplog, default_conf):
@@ -48,10 +47,10 @@ def test_load_strategy_base64(result, caplog, default_conf):
     default_conf.update({'strategy': 'SampleStrategy:{}'.format(encoded_string)})
 
     resolver = StrategyResolver(default_conf)
-    assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
+    assert 'rsi' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
     # Make sure strategy was loaded from base64 (using temp directory)!!
     assert log_has_re(r"Using resolved strategy SampleStrategy from '"
-                      + tempfile.gettempdir() + r"/.*/SampleStrategy\.py'\.\.\.", caplog)
+                      r".*(/|\\).*(/|\\)SampleStrategy\.py'\.\.\.", caplog)
 
 
 def test_load_strategy_invalid_directory(result, caplog, default_conf):
@@ -256,23 +255,23 @@ def test_strategy_override_use_sell_signal(caplog, default_conf):
         'strategy': 'DefaultStrategy',
     })
     resolver = StrategyResolver(default_conf)
-    assert not resolver.strategy.use_sell_signal
+    assert resolver.strategy.use_sell_signal
     assert isinstance(resolver.strategy.use_sell_signal, bool)
     # must be inserted to configuration
-    assert 'use_sell_signal' in default_conf['experimental']
+    assert 'use_sell_signal' in default_conf['ask_strategy']
-    assert not default_conf['experimental']['use_sell_signal']
+    assert default_conf['ask_strategy']['use_sell_signal']
 
     default_conf.update({
         'strategy': 'DefaultStrategy',
-        'experimental': {
+        'ask_strategy': {
-            'use_sell_signal': True,
+            'use_sell_signal': False,
         },
     })
     resolver = StrategyResolver(default_conf)
 
-    assert resolver.strategy.use_sell_signal
+    assert not resolver.strategy.use_sell_signal
     assert isinstance(resolver.strategy.use_sell_signal, bool)
-    assert log_has("Override strategy 'use_sell_signal' with value in config file: True.", caplog)
+    assert log_has("Override strategy 'use_sell_signal' with value in config file: False.", caplog)
 
 
 def test_strategy_override_use_sell_profit_only(caplog, default_conf):
@@ -284,12 +283,12 @@ def test_strategy_override_use_sell_profit_only(caplog, default_conf):
     assert not resolver.strategy.sell_profit_only
     assert isinstance(resolver.strategy.sell_profit_only, bool)
     # must be inserted to configuration
-    assert 'sell_profit_only' in default_conf['experimental']
+    assert 'sell_profit_only' in default_conf['ask_strategy']
-    assert not default_conf['experimental']['sell_profit_only']
+    assert not default_conf['ask_strategy']['sell_profit_only']
 
     default_conf.update({
         'strategy': 'DefaultStrategy',
-        'experimental': {
+        'ask_strategy': {
             'sell_profit_only': True,
         },
     })

tests/test_arguments.py

@@ -1,5 +1,7 @@
 # pragma pylint: disable=missing-docstring, C0103
 import argparse
+from pathlib import Path
+from unittest.mock import MagicMock
 
 import pytest
 
@@ -177,6 +179,44 @@ def test_plot_profit_options() -> None:
     assert pargs["db_url"] == "sqlite:///whatever.sqlite"
 
 
+def test_config_notallowed(mocker) -> None:
+    mocker.patch.object(Path, "is_file", MagicMock(return_value=False))
+    args = [
+        'create-userdir',
+    ]
+    pargs = Arguments(args).get_parsed_arg()
+
+    assert pargs["config"] is None
+
+    # When file exists:
+    mocker.patch.object(Path, "is_file", MagicMock(return_value=True))
+    args = [
+        'create-userdir',
+    ]
+    pargs = Arguments(args).get_parsed_arg()
+    # config is not added even if it exists, since create-userdir is in the notallowed list
+    assert pargs["config"] is None
+
+
+def test_config_notrequired(mocker) -> None:
+    mocker.patch.object(Path, "is_file", MagicMock(return_value=False))
+    args = [
+        'download-data',
+    ]
+    pargs = Arguments(args).get_parsed_arg()
+
+    assert pargs["config"] is None
+
+    # When file exists:
+    mocker.patch.object(Path, "is_file", MagicMock(return_value=True))
+    args = [
+        'download-data',
+    ]
+    pargs = Arguments(args).get_parsed_arg()
+    # config is added if it exists
+    assert pargs["config"] == ['config.json']
+
+
 def test_check_int_positive() -> None:
     assert check_int_positive("3") == 3
     assert check_int_positive("1") == 1

tests/test_configuration.py

@@ -14,6 +14,9 @@ from freqtrade.configuration import (Arguments, Configuration,
                                      validate_config_consistency)
 from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.config_validation import validate_config_schema
+from freqtrade.configuration.deprecated_settings import (check_conflicting_settings,
+                                                         process_deprecated_setting,
+                                                         process_temporary_deprecated_settings)
 from freqtrade.configuration.directory_operations import (create_datadir,
                                                           create_userdata_dir)
 from freqtrade.configuration.load_config import load_config_file
@@ -396,14 +399,14 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
     assert log_has('Using data directory: {} ...'.format("/foo/bar"), caplog)
-    assert log_has('Using user-data directory: {} ...'.format("/tmp/freqtrade"), caplog)
+    assert log_has('Using user-data directory: {} ...'.format(Path("/tmp/freqtrade")), caplog)
     assert 'user_data_dir' in config
 
     assert 'ticker_interval' in config
     assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
                    caplog)
 
-    assert 'position_stacking'in config
+    assert 'position_stacking' in config
     assert log_has('Parameter --enable-position-stacking detected ...', caplog)
 
     assert 'use_max_market_positions' in config
@@ -506,7 +509,8 @@ def test_check_exchange(default_conf, caplog) -> None:
     # Test an available exchange, supported by ccxt
     default_conf.get('exchange').update({'name': 'huobipro'})
     assert check_exchange(default_conf)
-    assert log_has_re(r"Exchange .* is supported by ccxt and .* not officially supported "
+    assert log_has_re(r"Exchange .* is known to the the ccxt library, available for the bot, "
+                      r"but not officially supported "
                       r"by the Freqtrade development team\. .*", caplog)
     caplog.clear()
 
@@ -520,16 +524,16 @@ def test_check_exchange(default_conf, caplog) -> None:
     # Test a 'bad' exchange with check_for_bad=False
     default_conf.get('exchange').update({'name': 'bitmex'})
     assert check_exchange(default_conf, False)
-    assert log_has_re(r"Exchange .* is supported by ccxt and .* not officially supported "
+    assert log_has_re(r"Exchange .* is known to the the ccxt library, available for the bot, "
+                      r"but not officially supported "
                       r"by the Freqtrade development team\. .*", caplog)
     caplog.clear()
 
     # Test an invalid exchange
     default_conf.get('exchange').update({'name': 'unknown_exchange'})
-
     with pytest.raises(
         OperationalException,
-        match=r'.*Exchange "unknown_exchange" is not supported by ccxt '
+        match=r'Exchange "unknown_exchange" is not known to the ccxt library '
               r'and therefore not available for the bot.*'
     ):
         check_exchange(default_conf)
@@ -648,9 +652,9 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None:
     x = create_userdata_dir('/tmp/bar', create_dir=True)
     assert md.call_count == 7
     assert md.call_args[1]['parents'] is False
-    assert log_has('Created user-data directory: /tmp/bar', caplog)
+    assert log_has(f'Created user-data directory: {Path("/tmp/bar")}', caplog)
     assert isinstance(x, Path)
-    assert str(x) == "/tmp/bar"
+    assert str(x) == str(Path("/tmp/bar"))
 
 
 def test_create_userdata_dir_exists(mocker, default_conf, caplog) -> None:
@@ -665,7 +669,8 @@ def test_create_userdata_dir_exists_exception(mocker, default_conf, caplog) -> N
     mocker.patch.object(Path, "is_dir", MagicMock(return_value=False))
     md = mocker.patch.object(Path, 'mkdir', MagicMock())
 
-    with pytest.raises(OperationalException, match=r'Directory `/tmp/bar` does not exist.*'):
+    with pytest.raises(OperationalException,
+                       match=r'Directory `.{1,2}tmp.{1,2}bar` does not exist.*'):
         create_userdata_dir('/tmp/bar',  create_dir=False)
     assert md.call_count == 0
 
@@ -896,3 +901,126 @@ def test_pairlist_resolving_fallback(mocker):
     assert config['pairs'] == ['ETH/BTC', 'XRP/BTC']
     assert config['exchange']['name'] == 'binance'
     assert config['datadir'] == str(Path.cwd() / "user_data/data/binance")
+
+
+@pytest.mark.parametrize("setting", [
+        ("ask_strategy", "use_sell_signal", True,
+         "experimental", "use_sell_signal", False),
+        ("ask_strategy", "sell_profit_only", False,
+         "experimental", "sell_profit_only", True),
+        ("ask_strategy", "ignore_roi_if_buy_signal", False,
+         "experimental", "ignore_roi_if_buy_signal", True),
+    ])
+def test_process_temporary_deprecated_settings(mocker, default_conf, setting, caplog):
+    patched_configuration_load_config_file(mocker, default_conf)
+
+    # Create sections for new and deprecated settings
+    # (they may not exist in the config)
+    default_conf[setting[0]] = {}
+    default_conf[setting[3]] = {}
+    # Assign new setting
+    default_conf[setting[0]][setting[1]] = setting[2]
+    # Assign deprecated setting
+    default_conf[setting[3]][setting[4]] = setting[5]
+
+    # New and deprecated settings are conflicting ones
+    with pytest.raises(OperationalException, match=r'DEPRECATED'):
+        process_temporary_deprecated_settings(default_conf)
+
+    caplog.clear()
+
+    # Delete new setting
+    del default_conf[setting[0]][setting[1]]
+
+    process_temporary_deprecated_settings(default_conf)
+    assert log_has_re('DEPRECATED', caplog)
+    # The value of the new setting shall have been set to the
+    # value of the deprecated one
+    assert default_conf[setting[0]][setting[1]] == setting[5]
+
+
+def test_check_conflicting_settings(mocker, default_conf, caplog):
+    patched_configuration_load_config_file(mocker, default_conf)
+
+    # Create sections for new and deprecated settings
+    # (they may not exist in the config)
+    default_conf['sectionA'] = {}
+    default_conf['sectionB'] = {}
+    # Assign new setting
+    default_conf['sectionA']['new_setting'] = 'valA'
+    # Assign deprecated setting
+    default_conf['sectionB']['deprecated_setting'] = 'valB'
+
+    # New and deprecated settings are conflicting ones
+    with pytest.raises(OperationalException, match=r'DEPRECATED'):
+        check_conflicting_settings(default_conf,
+                                   'sectionA', 'new_setting',
+                                   'sectionB', 'deprecated_setting')
+
+    caplog.clear()
+
+    # Delete new setting (deprecated exists)
+    del default_conf['sectionA']['new_setting']
+    check_conflicting_settings(default_conf,
+                               'sectionA', 'new_setting',
+                               'sectionB', 'deprecated_setting')
+    assert not log_has_re('DEPRECATED', caplog)
+    assert 'new_setting' not in default_conf['sectionA']
+
+    caplog.clear()
+
+    # Assign new setting
+    default_conf['sectionA']['new_setting'] = 'valA'
+    # Delete deprecated setting
+    del default_conf['sectionB']['deprecated_setting']
+    check_conflicting_settings(default_conf,
+                               'sectionA', 'new_setting',
+                               'sectionB', 'deprecated_setting')
+    assert not log_has_re('DEPRECATED', caplog)
+    assert default_conf['sectionA']['new_setting'] == 'valA'
+
+
+def test_process_deprecated_setting(mocker, default_conf, caplog):
+    patched_configuration_load_config_file(mocker, default_conf)
+
+    # Create sections for new and deprecated settings
+    # (they may not exist in the config)
+    default_conf['sectionA'] = {}
+    default_conf['sectionB'] = {}
+    # Assign new setting
+    default_conf['sectionA']['new_setting'] = 'valA'
+    # Assign deprecated setting
+    default_conf['sectionB']['deprecated_setting'] = 'valB'
+
+    # Both new and deprecated settings exists
+    process_deprecated_setting(default_conf,
+                               'sectionA', 'new_setting',
+                               'sectionB', 'deprecated_setting')
+    assert log_has_re('DEPRECATED', caplog)
+    # The value of the new setting shall have been set to the
+    # value of the deprecated one
+    assert default_conf['sectionA']['new_setting'] == 'valB'
+
+    caplog.clear()
+
+    # Delete new setting (deprecated exists)
+    del default_conf['sectionA']['new_setting']
+    process_deprecated_setting(default_conf,
+                               'sectionA', 'new_setting',
+                               'sectionB', 'deprecated_setting')
+    assert log_has_re('DEPRECATED', caplog)
+    # The value of the new setting shall have been set to the
+    # value of the deprecated one
+    assert default_conf['sectionA']['new_setting'] == 'valB'
+
+    caplog.clear()
+
+    # Assign new setting
+    default_conf['sectionA']['new_setting'] = 'valA'
+    # Delete deprecated setting
+    del default_conf['sectionB']['deprecated_setting']
+    process_deprecated_setting(default_conf,
+                               'sectionA', 'new_setting',
+                               'sectionB', 'deprecated_setting')
+    assert not log_has_re('DEPRECATED', caplog)
+    assert default_conf['sectionA']['new_setting'] == 'valA'

tests/test_freqtradebot.py

@@ -655,7 +655,8 @@ def test_create_trades_no_pairs_let(default_conf, ticker, limit_buy_order, fee,
 
     assert freqtrade.create_trades()
     assert not freqtrade.create_trades()
-    assert log_has("No currency pair in whitelist, but checking to sell open trades.", caplog)
+    assert log_has("No currency pair in active pair whitelist, "
+                   "but checking to sell open trades.", caplog)
 
 
 def test_create_trades_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee,
@@ -674,7 +675,7 @@ def test_create_trades_no_pairs_in_whitelist(default_conf, ticker, limit_buy_ord
     patch_get_signal(freqtrade)
 
     assert not freqtrade.create_trades()
-    assert log_has("Whitelist is empty.", caplog)
+    assert log_has("Active pair whitelist is empty.", caplog)
 
 
 def test_create_trades_no_signal(default_conf, fee, mocker) -> None:
@@ -1057,8 +1058,9 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None
     trade.open_order_id = None
     trade.stoploss_order_id = None
     trade.is_open = True
+    trades = [trade]
 
-    freqtrade.process_maybe_execute_sell(trade)
+    freqtrade.process_maybe_execute_sells(trades)
     assert trade.stoploss_order_id == '13434334'
     assert stoploss_limit.call_count == 1
     assert trade.is_open is True
@@ -1447,7 +1449,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
     # setting stoploss
     freqtrade.strategy.stoploss = -0.02
 
-    # setting stoploss_on_exchange_interval to 0 second
+    # setting stoploss_on_exchange_interval to 0 seconds
     freqtrade.strategy.order_types['stoploss_on_exchange_interval'] = 0
 
     patch_get_signal(freqtrade)
@@ -1518,26 +1520,27 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
                                                 stop_price=0.00002344 * 0.99)
 
 
-def test_process_maybe_execute_buy(mocker, default_conf, caplog) -> None:
+def test_process_maybe_execute_buys(mocker, default_conf, caplog) -> None:
+    caplog.set_level(logging.DEBUG)
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trades', MagicMock(return_value=False))
-    freqtrade.process_maybe_execute_buy()
+    freqtrade.process_maybe_execute_buys()
     assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog)
 
 
-def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> None:
+def test_process_maybe_execute_buys_exception(mocker, default_conf, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     mocker.patch(
         'freqtrade.freqtradebot.FreqtradeBot.create_trades',
         MagicMock(side_effect=DependencyException)
     )
-    freqtrade.process_maybe_execute_buy()
+    freqtrade.process_maybe_execute_buys()
     assert log_has('Unable to create trade: ', caplog)
 
 
-def test_process_maybe_execute_sell(mocker, default_conf, limit_buy_order, caplog) -> None:
+def test_process_maybe_execute_sells(mocker, default_conf, limit_buy_order, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True))
@@ -1549,7 +1552,8 @@ def test_process_maybe_execute_sell(mocker, default_conf, limit_buy_order, caplo
     trade = MagicMock()
     trade.open_order_id = '123'
     trade.open_fee = 0.001
-    assert not freqtrade.process_maybe_execute_sell(trade)
+    trades = [trade]
+    assert not freqtrade.process_maybe_execute_sells(trades)
     # Test amount not modified by fee-logic
     assert not log_has(
         'Applying fee to amount for Trade {} from 90.99181073 to 90.81'.format(trade), caplog
@@ -1557,24 +1561,25 @@ def test_process_maybe_execute_sell(mocker, default_conf, limit_buy_order, caplo
 
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81)
     # test amount modified by fee-logic
-    assert not freqtrade.process_maybe_execute_sell(trade)
+    assert not freqtrade.process_maybe_execute_sells(trades)
 
 
-def test_process_maybe_execute_sell_exception(mocker, default_conf,
+def test_process_maybe_execute_sells_exception(mocker, default_conf,
-                                              limit_buy_order, caplog) -> None:
+                                               limit_buy_order, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order)
 
     trade = MagicMock()
     trade.open_order_id = '123'
     trade.open_fee = 0.001
+    trades = [trade]
 
     # Test raise of DependencyException exception
     mocker.patch(
         'freqtrade.freqtradebot.FreqtradeBot.update_trade_state',
         side_effect=DependencyException()
     )
-    freqtrade.process_maybe_execute_sell(trade)
+    freqtrade.process_maybe_execute_sells(trades)
     assert log_has('Unable to sell trade: ', caplog)
 
 
@@ -1674,7 +1679,7 @@ def test_update_trade_state_exception(mocker, default_conf,
     # Test raise of OperationalException exception
     mocker.patch(
         'freqtrade.freqtradebot.FreqtradeBot.get_real_amount',
-        side_effect=OperationalException()
+        side_effect=DependencyException()
     )
     freqtrade.update_trade_state(trade)
     assert log_has('Could not update trade amount: ', caplog)
@@ -1768,8 +1773,6 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order,
 
 def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order,
                                      fee, markets, mocker) -> None:
-    default_conf.update({'experimental': {'use_sell_signal': True}})
-
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
@@ -1824,7 +1827,6 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order,
 def test_handle_trade_roi(default_conf, ticker, limit_buy_order,
                           fee, mocker, markets, caplog) -> None:
     caplog.set_level(logging.DEBUG)
-    default_conf.update({'experimental': {'use_sell_signal': True}})
 
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -1856,10 +1858,10 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order,
                    caplog)
 
 
-def test_handle_trade_experimental(
+def test_handle_trade_use_sell_signal(
         default_conf, ticker, limit_buy_order, fee, mocker, markets, caplog) -> None:
+    # use_sell_signal is True buy default
     caplog.set_level(logging.DEBUG)
-    default_conf.update({'experimental': {'use_sell_signal': True}})
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
@@ -1911,11 +1913,12 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order,
     trade.update(limit_sell_order)
     assert trade.is_open is False
 
-    with pytest.raises(ValueError, match=r'.*closed trade.*'):
+    with pytest.raises(DependencyException, match=r'.*closed trade.*'):
         freqtrade.handle_trade(trade)
 
 
-def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fee, mocker) -> None:
+def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, open_trade,
+                                   fee, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
     cancel_order_mock = MagicMock()
     patch_exchange(mocker)
@@ -1928,31 +1931,18 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    trade_buy = Trade(
+    Trade.session.add(open_trade)
-        pair='ETH/BTC',
-        open_rate=0.00001099,
-        exchange='bittrex',
-        open_order_id='123456789',
-        amount=90.99181073,
-        fee_open=0.0,
-        fee_close=0.0,
-        stake_amount=1,
-        open_date=arrow.utcnow().shift(minutes=-601).datetime,
-        is_open=True
-    )
-
-    Trade.session.add(trade_buy)
 
     # check it does cancel buy orders over the time limit
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 1
     assert rpc_mock.call_count == 1
-    trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all()
+    trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
     assert nb_trades == 0
 
 
-def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old,
+def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, open_trade,
                                     fee, mocker, caplog) -> None:
     """ Handle Buy order cancelled on exchange"""
     rpc_mock = patch_RPCManager(mocker)
@@ -1968,32 +1958,19 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old,
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    trade_buy = Trade(
+    Trade.session.add(open_trade)
-        pair='ETH/BTC',
-        open_rate=0.00001099,
-        exchange='bittrex',
-        open_order_id='123456789',
-        amount=90.99181073,
-        fee_open=0.0,
-        fee_close=0.0,
-        stake_amount=1,
-        open_date=arrow.utcnow().shift(minutes=-601).datetime,
-        is_open=True
-    )
-
-    Trade.session.add(trade_buy)
 
     # check it does cancel buy orders over the time limit
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 0
     assert rpc_mock.call_count == 1
-    trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all()
+    trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
     assert nb_trades == 0
     assert log_has_re("Buy order canceled on Exchange for Trade.*", caplog)
 
 
-def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old,
+def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, open_trade,
                                              fee, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
     cancel_order_mock = MagicMock()
@@ -2008,31 +1985,19 @@ def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_ord
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    trade_buy = Trade(
+    Trade.session.add(open_trade)
-        pair='ETH/BTC',
-        open_rate=0.00001099,
-        exchange='bittrex',
-        open_order_id='123456789',
-        amount=90.99181073,
-        fee_open=0.0,
-        fee_close=0.0,
-        stake_amount=1,
-        open_date=arrow.utcnow().shift(minutes=-601).datetime,
-        is_open=True
-    )
-
-    Trade.session.add(trade_buy)
 
     # check it does cancel buy orders over the time limit
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 0
     assert rpc_mock.call_count == 0
-    trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all()
+    trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
     assert nb_trades == 1
 
 
-def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None:
+def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker,
+                                    open_trade) -> None:
     rpc_mock = patch_RPCManager(mocker)
     cancel_order_mock = MagicMock()
     patch_exchange(mocker)
@@ -2044,30 +2009,20 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old,
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    trade_sell = Trade(
+    open_trade.open_date = arrow.utcnow().shift(hours=-5).datetime
-        pair='ETH/BTC',
+    open_trade.close_date = arrow.utcnow().shift(minutes=-601).datetime
-        open_rate=0.00001099,
+    open_trade.is_open = False
-        exchange='bittrex',
-        open_order_id='123456789',
-        amount=90.99181073,
-        fee_open=0.0,
-        fee_close=0.0,
-        stake_amount=1,
-        open_date=arrow.utcnow().shift(hours=-5).datetime,
-        close_date=arrow.utcnow().shift(minutes=-601).datetime,
-        is_open=False
-    )
 
-    Trade.session.add(trade_sell)
+    Trade.session.add(open_trade)
 
     # check it does cancel sell orders over the time limit
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 1
     assert rpc_mock.call_count == 1
-    assert trade_sell.is_open is True
+    assert open_trade.is_open is True
 
 
-def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old,
+def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, open_trade,
                                      mocker, caplog) -> None:
     """ Handle sell order cancelled on exchange"""
     rpc_mock = patch_RPCManager(mocker)
@@ -2082,34 +2037,24 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old,
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    trade_sell = Trade(
+    open_trade.open_date = arrow.utcnow().shift(hours=-5).datetime
-        pair='ETH/BTC',
+    open_trade.close_date = arrow.utcnow().shift(minutes=-601).datetime
-        open_rate=0.00001099,
+    open_trade.is_open = False
-        exchange='bittrex',
-        open_order_id='123456789',
-        amount=90.99181073,
-        fee_open=0.0,
-        fee_close=0.0,
-        stake_amount=1,
-        open_date=arrow.utcnow().shift(hours=-5).datetime,
-        close_date=arrow.utcnow().shift(minutes=-601).datetime,
-        is_open=False
-    )
 
-    Trade.session.add(trade_sell)
+    Trade.session.add(open_trade)
 
     # check it does cancel sell orders over the time limit
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 0
     assert rpc_mock.call_count == 1
-    assert trade_sell.is_open is True
+    assert open_trade.is_open is True
     assert log_has_re("Sell order canceled on exchange for Trade.*", caplog)
 
 
 def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial,
-                                       mocker) -> None:
+                                       open_trade, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
-    cancel_order_mock = MagicMock()
+    cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
@@ -2119,33 +2064,97 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    trade_buy = Trade(
+    Trade.session.add(open_trade)
-        pair='ETH/BTC',
-        open_rate=0.00001099,
-        exchange='bittrex',
-        open_order_id='123456789',
-        amount=90.99181073,
-        fee_open=0.0,
-        fee_close=0.0,
-        stake_amount=1,
-        open_date=arrow.utcnow().shift(minutes=-601).datetime,
-        is_open=True
-    )
-
-    Trade.session.add(trade_buy)
 
     # check it does cancel buy orders over the time limit
     # note this is for a partially-complete buy order
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 1
     assert rpc_mock.call_count == 1
-    trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all()
+    trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     assert trades[0].amount == 23.0
-    assert trades[0].stake_amount == trade_buy.open_rate * trades[0].amount
+    assert trades[0].stake_amount == open_trade.open_rate * trades[0].amount
 
 
-def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -> None:
+def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, caplog, fee,
+                                           limit_buy_order_old_partial, trades_for_order,
+                                           limit_buy_order_old_partial_canceled, mocker) -> None:
+    rpc_mock = patch_RPCManager(mocker)
+    cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled)
+    patch_exchange(mocker)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_ticker=ticker,
+        get_order=MagicMock(return_value=limit_buy_order_old_partial),
+        cancel_order=cancel_order_mock,
+        get_trades_for_order=MagicMock(return_value=trades_for_order),
+    )
+    freqtrade = FreqtradeBot(default_conf)
+
+    assert open_trade.amount == limit_buy_order_old_partial['amount']
+
+    open_trade.fee_open = fee()
+    open_trade.fee_close = fee()
+    Trade.session.add(open_trade)
+    # cancelling a half-filled order should update the amount to the bought amount
+    # and apply fees if necessary.
+    freqtrade.check_handle_timedout()
+
+    assert log_has_re(r"Applying fee on amount for Trade.* Order", caplog)
+
+    assert cancel_order_mock.call_count == 1
+    assert rpc_mock.call_count == 1
+    trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
+    assert len(trades) == 1
+    # Verify that tradehas been updated
+    assert trades[0].amount == (limit_buy_order_old_partial['amount'] -
+                                limit_buy_order_old_partial['remaining']) - 0.0001
+    assert trades[0].open_order_id is None
+    assert trades[0].fee_open == 0
+
+
+def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee,
+                                              limit_buy_order_old_partial, trades_for_order,
+                                              limit_buy_order_old_partial_canceled, mocker) -> None:
+    rpc_mock = patch_RPCManager(mocker)
+    cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled)
+    patch_exchange(mocker)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_ticker=ticker,
+        get_order=MagicMock(return_value=limit_buy_order_old_partial),
+        cancel_order=cancel_order_mock,
+        get_trades_for_order=MagicMock(return_value=trades_for_order),
+    )
+    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount',
+                 MagicMock(side_effect=DependencyException))
+    freqtrade = FreqtradeBot(default_conf)
+
+    assert open_trade.amount == limit_buy_order_old_partial['amount']
+
+    open_trade.fee_open = fee()
+    open_trade.fee_close = fee()
+    Trade.session.add(open_trade)
+    # cancelling a half-filled order should update the amount to the bought amount
+    # and apply fees if necessary.
+    freqtrade.check_handle_timedout()
+
+    assert log_has_re(r"Could not update trade amount: .*", caplog)
+
+    assert cancel_order_mock.call_count == 1
+    assert rpc_mock.call_count == 1
+    trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
+    assert len(trades) == 1
+    # Verify that tradehas been updated
+
+    assert trades[0].amount == (limit_buy_order_old_partial['amount'] -
+                                limit_buy_order_old_partial['remaining'])
+    assert trades[0].open_order_id is None
+    assert trades[0].fee_open == fee()
+
+
+def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocker, caplog) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     cancel_order_mock = MagicMock()
@@ -2163,34 +2172,20 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    open_date = arrow.utcnow().shift(minutes=-601)
+    Trade.session.add(open_trade)
-    trade_buy = Trade(
-        pair='ETH/BTC',
-        open_rate=0.00001099,
-        exchange='bittrex',
-        open_order_id='123456789',
-        amount=90.99181073,
-        fee_open=0.0,
-        fee_close=0.0,
-        stake_amount=1,
-        open_date=open_date.datetime,
-        is_open=True
-    )
-
-    Trade.session.add(trade_buy)
 
     freqtrade.check_handle_timedout()
     assert log_has_re(r"Cannot query order for Trade\(id=1, pair=ETH/BTC, amount=90.99181073, "
                       r"open_rate=0.00001099, open_since="
-                      f"{open_date.strftime('%Y-%m-%d %H:%M:%S')}"
+                      f"{open_trade.open_date.strftime('%Y-%m-%d %H:%M:%S')}"
                       r"\) due to Traceback \(most recent call last\):\n*",
                       caplog)
 
 
-def test_handle_timedout_limit_buy(mocker, default_conf) -> None:
+def test_handle_timedout_limit_buy(mocker, default_conf, limit_buy_order) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
-    cancel_order_mock = MagicMock()
+    cancel_order_mock = MagicMock(return_value=limit_buy_order)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         cancel_order=cancel_order_mock
@@ -2200,13 +2195,14 @@ def test_handle_timedout_limit_buy(mocker, default_conf) -> None:
 
     Trade.session = MagicMock()
     trade = MagicMock()
-    order = {'remaining': 1,
+    limit_buy_order['remaining'] = limit_buy_order['amount']
-             'amount': 1}
+    assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
-    assert freqtrade.handle_timedout_limit_buy(trade, order)
+    assert cancel_order_mock.call_count == 1
+
+    cancel_order_mock.reset_mock()
+    limit_buy_order['amount'] = 2
+    assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
     assert cancel_order_mock.call_count == 1
-    order['amount'] = 2
-    assert not freqtrade.handle_timedout_limit_buy(trade, order)
-    assert cancel_order_mock.call_count == 2
 
 
 def test_handle_timedout_limit_sell(mocker, default_conf) -> None:
@@ -2418,13 +2414,6 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf,
     default_conf['exchange']['name'] = 'binance'
     rpc_mock = patch_RPCManager(mocker)
     patch_exchange(mocker)
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        get_ticker=ticker,
-        get_fee=fee,
-        markets=PropertyMock(return_value=markets)
-    )
-
     stoploss_limit = MagicMock(return_value={
         'id': 123,
         'info': {
@@ -2433,11 +2422,16 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf,
     })
 
     cancel_order = MagicMock(return_value=True)
-
+    mocker.patch.multiple(
-    mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y)
+        'freqtrade.exchange.Exchange',
-    mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y)
+        get_ticker=ticker,
-    mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit)
+        get_fee=fee,
-    mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order)
+        markets=PropertyMock(return_value=markets),
+        symbol_amount_prec=lambda s, x, y: y,
+        symbol_price_prec=lambda s, x, y: y,
+        stoploss_limit=stoploss_limit,
+        cancel_order=cancel_order,
+    )
 
     freqtrade = FreqtradeBot(default_conf)
     freqtrade.strategy.order_types['stoploss_on_exchange'] = True
@@ -2448,8 +2442,9 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf,
 
     trade = Trade.query.first()
     assert trade
+    trades = [trade]
 
-    freqtrade.process_maybe_execute_sell(trade)
+    freqtrade.process_maybe_execute_sells(trades)
 
     # Increase the price and sell it
     mocker.patch.multiple(
@@ -2477,7 +2472,9 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
         'freqtrade.exchange.Exchange',
         get_ticker=ticker,
         get_fee=fee,
-        markets=PropertyMock(return_value=markets)
+        markets=PropertyMock(return_value=markets),
+        symbol_amount_prec=lambda s, x, y: y,
+        symbol_price_prec=lambda s, x, y: y,
     )
 
     stoploss_limit = MagicMock(return_value={
@@ -2487,8 +2484,6 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
         }
     })
 
-    mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y)
-    mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y)
     mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit)
 
     freqtrade = FreqtradeBot(default_conf)
@@ -2498,7 +2493,8 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
     # Create some test data
     freqtrade.create_trades()
     trade = Trade.query.first()
-    freqtrade.process_maybe_execute_sell(trade)
+    trades = [trade]
+    freqtrade.process_maybe_execute_sells(trades)
     assert trade
     assert trade.stoploss_order_id == '123'
     assert trade.open_order_id is None
@@ -2526,13 +2522,122 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf,
     })
     mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_limit_executed)
 
-    freqtrade.process_maybe_execute_sell(trade)
+    freqtrade.process_maybe_execute_sells(trades)
     assert trade.stoploss_order_id is None
     assert trade.is_open is False
     assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value
     assert rpc_mock.call_count == 2
 
 
+def test_may_execute_sell_stoploss_on_exchange_multi(default_conf,
+                                                     ticker, fee,
+                                                     limit_buy_order,
+                                                     markets, mocker) -> None:
+    """
+    Tests workflow of selling stoploss_on_exchange.
+    Sells
+    * first trade as stoploss
+    * 2nd trade is kept
+    * 3rd trade is sold via sell-signal
+    """
+    default_conf['max_open_trades'] = 3
+    default_conf['exchange']['name'] = 'binance'
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+
+    stoploss_limit = {
+        'id': 123,
+        'info': {}
+    }
+    stoploss_order_open = {
+        "id": "123",
+        "timestamp": 1542707426845,
+        "datetime": "2018-11-20T09:50:26.845Z",
+        "lastTradeTimestamp": None,
+        "symbol": "BTC/USDT",
+        "type": "stop_loss_limit",
+        "side": "sell",
+        "price": 1.08801,
+        "amount": 90.99181074,
+        "cost": 0.0,
+        "average": 0.0,
+        "filled": 0.0,
+        "remaining": 0.0,
+        "status": "open",
+        "fee": None,
+        "trades": None
+    }
+    stoploss_order_closed = stoploss_order_open.copy()
+    stoploss_order_closed['status'] = 'closed'
+    # Sell first trade based on stoploss, keep 2nd and 3rd trade open
+    stoploss_order_mock = MagicMock(
+        side_effect=[stoploss_order_closed, stoploss_order_open, stoploss_order_open])
+    # Sell 3rd trade (not called for the first trade)
+    should_sell_mock = MagicMock(side_effect=[
+        SellCheckTuple(sell_flag=False, sell_type=SellType.NONE),
+        SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL)]
+    )
+    cancel_order_mock = MagicMock()
+    mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        get_ticker=ticker,
+        get_fee=fee,
+        markets=PropertyMock(return_value=markets),
+        symbol_amount_prec=lambda s, x, y: y,
+        symbol_price_prec=lambda s, x, y: y,
+        get_order=stoploss_order_mock,
+        cancel_order=cancel_order_mock,
+    )
+
+    wallets_mock = MagicMock()
+    mocker.patch.multiple(
+        'freqtrade.freqtradebot.FreqtradeBot',
+        create_stoploss_order=MagicMock(return_value=True),
+        update_trade_state=MagicMock(),
+        _notify_sell=MagicMock(),
+    )
+    mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock)
+    mocker.patch("freqtrade.wallets.Wallets.update", wallets_mock)
+
+    freqtrade = FreqtradeBot(default_conf)
+    freqtrade.strategy.order_types['stoploss_on_exchange'] = True
+    # Switch ordertype to market to close trade immediately
+    freqtrade.strategy.order_types['sell'] = 'market'
+    patch_get_signal(freqtrade)
+
+    # Create some test data
+    freqtrade.create_trades()
+    wallets_mock.reset_mock()
+    Trade.session = MagicMock()
+
+    trades = Trade.query.all()
+    # Make sure stoploss-order is open and trade is bought (since we mock update_trade_state)
+    for trade in trades:
+        trade.stoploss_order_id = 3
+        trade.open_order_id = None
+
+    freqtrade.process_maybe_execute_sells(trades)
+    assert should_sell_mock.call_count == 2
+
+    # Only order for 3rd trade needs to be cancelled
+    assert cancel_order_mock.call_count == 1
+    # Wallets should only be called once per sell cycle
+    assert wallets_mock.call_count == 1
+
+    trade = trades[0]
+    assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value
+    assert not trade.is_open
+
+    trade = trades[1]
+    assert not trade.sell_reason
+    assert trade.is_open
+
+    trade = trades[2]
+    assert trade.sell_reason == SellType.SELL_SIGNAL.value
+    assert not trade.is_open
+
+
 def test_execute_sell_market_order(default_conf, ticker, fee,
                                    ticker_sell_up, markets, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
@@ -2600,7 +2705,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
     )
-    default_conf['experimental'] = {
+    default_conf['ask_strategy'] = {
         'use_sell_signal': True,
         'sell_profit_only': True,
     }
@@ -2632,7 +2737,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
     )
-    default_conf['experimental'] = {
+    default_conf['ask_strategy'] = {
         'use_sell_signal': True,
         'sell_profit_only': False,
     }
@@ -2662,7 +2767,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
     )
-    default_conf['experimental'] = {
+    default_conf['ask_strategy'] = {
         'use_sell_signal': True,
         'sell_profit_only': True,
     }
@@ -2692,7 +2797,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
     )
-    default_conf['experimental'] = {
+    default_conf['ask_strategy'] = {
         'use_sell_signal': True,
         'sell_profit_only': False,
     }
@@ -2761,7 +2866,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
     )
-    default_conf['experimental'] = {
+    default_conf['ask_strategy'] = {
         'ignore_roi_if_buy_signal': True
     }
     freqtrade = FreqtradeBot(default_conf)
@@ -3029,7 +3134,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order,
         get_fee=fee,
         markets=PropertyMock(return_value=markets)
     )
-    default_conf['experimental'] = {
+    default_conf['ask_strategy'] = {
         'ignore_roi_if_buy_signal': False
     }
     freqtrade = FreqtradeBot(default_conf)
@@ -3251,7 +3356,7 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_
     patch_get_signal(freqtrade)
 
     # Amount does not change
-    with pytest.raises(OperationalException, match=r"Half bought\? Amounts don't match"):
+    with pytest.raises(DependencyException, match=r"Half bought\? Amounts don't match"):
         freqtrade.get_real_amount(trade, limit_buy_order)
 
 
@@ -3543,3 +3648,27 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker):
     ftbot = get_patched_freqtradebot(mocker, edge_conf)
     ftbot.startup()
     assert reinit_mock.call_count == 0
+
+
+def test_process_i_am_alive(default_conf, mocker, caplog):
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+
+    ftbot = get_patched_freqtradebot(mocker, default_conf)
+    message = r"Bot heartbeat\. PID=.*"
+    ftbot.process()
+    assert log_has_re(message, caplog)
+    assert ftbot._heartbeat_msg != 0
+
+    caplog.clear()
+    # Message is not shown before interval is up
+    ftbot.process()
+    assert not log_has_re(message, caplog)
+
+    caplog.clear()
+    # Set clock - 70 seconds
+    ftbot._heartbeat_msg -= 70
+
+    ftbot.process()
+    assert log_has_re(message, caplog)

tests/test_indicator_helpers.py

@@ -1,15 +0,0 @@
-# pragma pylint: disable=missing-docstring
-
-import pandas as pd
-
-from freqtrade.indicator_helpers import went_down, went_up
-
-
-def test_went_up():
-    series = pd.Series([1, 2, 3, 1])
-    assert went_up(series).equals(pd.Series([False, True, True, False]))
-
-
-def test_went_down():
-    series = pd.Series([1, 2, 3, 1])
-    assert went_down(series).equals(pd.Series([False, False, False, True]))

tests/test_misc.py

@@ -7,7 +7,7 @@ from unittest.mock import MagicMock
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.data.history import pair_data_filename
 from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json,
-                            file_load_json, format_ms_time, shorten_date)
+                            file_load_json, format_ms_time, plural, shorten_date)
 
 
 def test_shorten_date() -> None:
@@ -69,3 +69,35 @@ def test_format_ms_time() -> None:
     # Date 2017-12-13 08:02:01
     date_in_epoch_ms = 1513152121000
     assert format_ms_time(date_in_epoch_ms) == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S')
+
+
+def test_plural() -> None:
+    assert plural(0, "page") == "pages"
+    assert plural(0.0, "page") == "pages"
+    assert plural(1, "page") == "page"
+    assert plural(1.0, "page") == "page"
+    assert plural(2, "page") == "pages"
+    assert plural(2.0, "page") == "pages"
+    assert plural(-1, "page") == "page"
+    assert plural(-1.0, "page") == "page"
+    assert plural(-2, "page") == "pages"
+    assert plural(-2.0, "page") == "pages"
+    assert plural(0.5, "page") == "pages"
+    assert plural(1.5, "page") == "pages"
+    assert plural(-0.5, "page") == "pages"
+    assert plural(-1.5, "page") == "pages"
+
+    assert plural(0, "ox", "oxen") == "oxen"
+    assert plural(0.0, "ox", "oxen") == "oxen"
+    assert plural(1, "ox", "oxen") == "ox"
+    assert plural(1.0, "ox", "oxen") == "ox"
+    assert plural(2, "ox", "oxen") == "oxen"
+    assert plural(2.0, "ox", "oxen") == "oxen"
+    assert plural(-1, "ox", "oxen") == "ox"
+    assert plural(-1.0, "ox", "oxen") == "ox"
+    assert plural(-2, "ox", "oxen") == "oxen"
+    assert plural(-2.0, "ox", "oxen") == "oxen"
+    assert plural(0.5, "ox", "oxen") == "oxen"
+    assert plural(1.5, "ox", "oxen") == "oxen"
+    assert plural(-0.5, "ox", "oxen") == "oxen"
+    assert plural(-1.5, "ox", "oxen") == "oxen"

tests/test_plotting.py

@@ -53,11 +53,11 @@ def test_init_plotscript(default_conf, mocker, testdatadir):
     assert "trades" in ret
     assert "pairs" in ret
 
-    default_conf['pairs'] = ["POWR/BTC", "XLM/BTC"]
+    default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"]
     ret = init_plotscript(default_conf)
     assert "tickers" in ret
-    assert "POWR/BTC" in ret["tickers"]
+    assert "TRX/BTC" in ret["tickers"]
-    assert "XLM/BTC" in ret["tickers"]
+    assert "ADA/BTC" in ret["tickers"]
 
 
 def test_add_indicators(default_conf, testdatadir, caplog):
@@ -197,8 +197,7 @@ def test_generate_candlestick_graph_no_trades(default_conf, mocker, testdatadir)
     # All buy-signals should be plotted
     assert int(data.sell.sum()) == len(sell.x)
 
-    assert find_trace_in_fig_data(figure.data, "BB lower")
+    assert find_trace_in_fig_data(figure.data, "Bollinger Band")
-    assert find_trace_in_fig_data(figure.data, "BB upper")
 
     assert row_mock.call_count == 2
     assert trades_mock.call_count == 1
@@ -215,11 +214,12 @@ def test_generate_plot_file(mocker, caplog):
     store_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html",
                     directory=Path("user_data/plots"))
 
+    expected_fn = str(Path("user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html"))
     assert plot_mock.call_count == 1
     assert plot_mock.call_args[0][0] == fig
     assert (plot_mock.call_args_list[0][1]['filename']
-            == "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html")
+            == expected_fn)
-    assert log_has("Stored plot as user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html",
+    assert log_has(f"Stored plot as {expected_fn}",
                    caplog)
 
 
@@ -228,18 +228,18 @@ def test_add_profit(testdatadir):
     bt_data = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
 
-    df = history.load_pair_history(pair="POWR/BTC", ticker_interval='5m',
+    df = history.load_pair_history(pair="TRX/BTC", ticker_interval='5m',
                                    datadir=testdatadir, timerange=timerange)
     fig = generate_empty_figure()
 
     cum_profits = create_cum_profit(df.set_index('date'),
-                                    bt_data[bt_data["pair"] == 'POWR/BTC'],
+                                    bt_data[bt_data["pair"] == 'TRX/BTC'],
-                                    "cum_profits")
+                                    "cum_profits", timeframe="5m")
 
     fig1 = add_profit(fig, row=2, data=cum_profits, column='cum_profits', name='Profits')
     figure = fig1.layout.figure
     profits = find_trace_in_fig_data(figure.data, "Profits")
-    assert isinstance(profits, go.Scattergl)
+    assert isinstance(profits, go.Scatter)
     assert profits.yaxis == "y2"
 
 
@@ -247,7 +247,7 @@ def test_generate_profit_graph(testdatadir):
     filename = testdatadir / "backtest-result_test.json"
     trades = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
-    pairs = ["POWR/BTC", "XLM/BTC"]
+    pairs = ["TRX/BTC", "ADA/BTC"]
 
     tickers = history.load_data(datadir=testdatadir,
                                 pairs=pairs,
@@ -256,7 +256,7 @@ def test_generate_profit_graph(testdatadir):
                                 )
     trades = trades[trades['pair'].isin(pairs)]
 
-    fig = generate_profit_graph(pairs, tickers, trades)
+    fig = generate_profit_graph(pairs, tickers, trades, timeframe="5m")
     assert isinstance(fig, go.Figure)
 
     assert fig.layout.title.text == "Freqtrade Profit plot"
@@ -268,14 +268,14 @@ def test_generate_profit_graph(testdatadir):
     assert len(figure.data) == 4
 
     avgclose = find_trace_in_fig_data(figure.data, "Avg close price")
-    assert isinstance(avgclose, go.Scattergl)
+    assert isinstance(avgclose, go.Scatter)
 
     profit = find_trace_in_fig_data(figure.data, "Profit")
-    assert isinstance(profit, go.Scattergl)
+    assert isinstance(profit, go.Scatter)
 
     for pair in pairs:
         profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}")
-        assert isinstance(profit_pair, go.Scattergl)
+        assert isinstance(profit_pair, go.Scatter)
 
 
 def test_start_plot_dataframe(mocker):

tests/test_timerange.py

@@ -5,9 +5,6 @@ from freqtrade.configuration import TimeRange
 
 
 def test_parse_timerange_incorrect() -> None:
-    assert TimeRange(None, 'line', 0, -200) == TimeRange.parse_timerange('-200')
-    assert TimeRange('line', None, 200, 0) == TimeRange.parse_timerange('200-')
-    assert TimeRange('index', 'index', 200, 500) == TimeRange.parse_timerange('200-500')
 
     assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-')
     assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522')
@@ -20,9 +17,14 @@ def test_parse_timerange_incorrect() -> None:
     timerange = TimeRange.parse_timerange('1231006505-1233360000')
     assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange
 
-    # TODO: Find solution for the following case (passing timestamp in ms)
     timerange = TimeRange.parse_timerange('1231006505000-1233360000000')
-    assert TimeRange('date', 'date', 1231006505, 1233360000) != timerange
+    assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange
+
+    timerange = TimeRange.parse_timerange('1231006505000-')
+    assert TimeRange('date', None, 1231006505, 0) == timerange
+
+    timerange = TimeRange.parse_timerange('-1231006505000')
+    assert TimeRange(None, 'date', 0, 1231006505) == timerange
 
     with pytest.raises(Exception, match=r'Incorrect syntax.*'):
         TimeRange.parse_timerange('-')

tests/test_utils.py

@@ -7,7 +7,8 @@ import pytest
 from freqtrade import OperationalException
 from freqtrade.state import RunMode
 from freqtrade.utils import (setup_utils_configuration, start_create_userdir,
-                             start_download_data, start_list_exchanges)
+                             start_download_data, start_list_exchanges,
+                             start_list_markets, start_list_timeframes)
 from tests.conftest import get_args, log_has, patch_exchange
 
 
@@ -31,7 +32,7 @@ def test_list_exchanges(capsys):
 
     start_list_exchanges(get_args(args))
     captured = capsys.readouterr()
-    assert re.match(r"Exchanges supported by ccxt and available.*", captured.out)
+    assert re.match(r"Exchanges available for Freqtrade.*", captured.out)
     assert re.match(r".*binance,.*", captured.out)
     assert re.match(r".*bittrex,.*", captured.out)
 
@@ -43,10 +44,366 @@ def test_list_exchanges(capsys):
 
     start_list_exchanges(get_args(args))
     captured = capsys.readouterr()
-    assert not re.match(r"Exchanges supported by ccxt and available.*", captured.out)
     assert re.search(r"^binance$", captured.out, re.MULTILINE)
     assert re.search(r"^bittrex$", captured.out, re.MULTILINE)
 
+    # Test with --all
+    args = [
+        "list-exchanges",
+        "--all",
+    ]
+
+    start_list_exchanges(get_args(args))
+    captured = capsys.readouterr()
+    assert re.match(r"All exchanges supported by the ccxt library.*", captured.out)
+    assert re.match(r".*binance,.*", captured.out)
+    assert re.match(r".*bittrex,.*", captured.out)
+    assert re.match(r".*bitmex,.*", captured.out)
+
+    # Test with --one-column --all
+    args = [
+        "list-exchanges",
+        "--one-column",
+        "--all",
+    ]
+
+    start_list_exchanges(get_args(args))
+    captured = capsys.readouterr()
+    assert re.search(r"^binance$", captured.out, re.MULTILINE)
+    assert re.search(r"^bittrex$", captured.out, re.MULTILINE)
+    assert re.search(r"^bitmex$", captured.out, re.MULTILINE)
+
+
+def test_list_timeframes(mocker, capsys):
+
+    api_mock = MagicMock()
+    api_mock.timeframes = {'1m': 'oneMin',
+                           '5m': 'fiveMin',
+                           '30m': 'thirtyMin',
+                           '1h': 'hour',
+                           '1d': 'day',
+                           }
+    patch_exchange(mocker, api_mock=api_mock)
+    args = [
+        "list-timeframes",
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    with pytest.raises(OperationalException,
+                       match=r"This command requires a configured exchange.*"):
+        start_list_timeframes(pargs)
+
+    # Test with --config config.json.example
+    args = [
+        '--config', 'config.json.example',
+        "list-timeframes",
+    ]
+    start_list_timeframes(get_args(args))
+    captured = capsys.readouterr()
+    assert re.match("Timeframes available for the exchange `Bittrex`: "
+                    "1m, 5m, 30m, 1h, 1d",
+                    captured.out)
+
+    # Test with --exchange bittrex
+    args = [
+        "list-timeframes",
+        "--exchange", "bittrex",
+    ]
+    start_list_timeframes(get_args(args))
+    captured = capsys.readouterr()
+    assert re.match("Timeframes available for the exchange `Bittrex`: "
+                    "1m, 5m, 30m, 1h, 1d",
+                    captured.out)
+
+    api_mock.timeframes = {'1m': '1m',
+                           '5m': '5m',
+                           '15m': '15m',
+                           '30m': '30m',
+                           '1h': '1h',
+                           '6h': '6h',
+                           '12h': '12h',
+                           '1d': '1d',
+                           '3d': '3d',
+                           }
+    patch_exchange(mocker, api_mock=api_mock, id='binance')
+    # Test with --exchange binance
+    args = [
+        "list-timeframes",
+        "--exchange", "binance",
+    ]
+    start_list_timeframes(get_args(args))
+    captured = capsys.readouterr()
+    assert re.match("Timeframes available for the exchange `Binance`: "
+                    "1m, 5m, 15m, 30m, 1h, 6h, 12h, 1d, 3d",
+                    captured.out)
+
+    # Test with --one-column
+    args = [
+        '--config', 'config.json.example',
+        "list-timeframes",
+        "--one-column",
+    ]
+    start_list_timeframes(get_args(args))
+    captured = capsys.readouterr()
+    assert re.search(r"^1m$", captured.out, re.MULTILINE)
+    assert re.search(r"^5m$", captured.out, re.MULTILINE)
+    assert re.search(r"^1h$", captured.out, re.MULTILINE)
+    assert re.search(r"^1d$", captured.out, re.MULTILINE)
+
+    # Test with --exchange binance --one-column
+    args = [
+        "list-timeframes",
+        "--exchange", "binance",
+        "--one-column",
+    ]
+    start_list_timeframes(get_args(args))
+    captured = capsys.readouterr()
+    assert re.search(r"^1m$", captured.out, re.MULTILINE)
+    assert re.search(r"^5m$", captured.out, re.MULTILINE)
+    assert re.search(r"^1h$", captured.out, re.MULTILINE)
+    assert re.search(r"^1d$", captured.out, re.MULTILINE)
+
+
+def test_list_markets(mocker, markets, capsys):
+
+    api_mock = MagicMock()
+    api_mock.markets = markets
+    patch_exchange(mocker, api_mock=api_mock)
+
+    # Test with no --config
+    args = [
+        "list-markets",
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    with pytest.raises(OperationalException,
+                       match=r"This command requires a configured exchange.*"):
+        start_list_markets(pargs, False)
+
+    # Test with --config config.json.example
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 8 active markets: "
+            "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/USD, LTC/USDT, TKN/BTC, XLTCUSDT.\n"
+            in captured.out)
+
+    patch_exchange(mocker, api_mock=api_mock, id="binance")
+    # Test with --exchange
+    args = [
+        "list-markets",
+        "--exchange", "binance"
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_list_markets(pargs, False)
+    captured = capsys.readouterr()
+    assert re.match("\nExchange Binance has 8 active markets:\n",
+                    captured.out)
+
+    patch_exchange(mocker, api_mock=api_mock, id="bittrex")
+    # Test with --all: all markets
+    args = [
+        '--config', 'config.json.example',
+        "list-markets", "--all",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 11 markets: "
+            "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, LTC/USDT, NEO/BTC, "
+            "TKN/BTC, XLTCUSDT, XRP/BTC.\n"
+            in captured.out)
+
+    # Test list-pairs subcommand: active pairs
+    args = [
+        '--config', 'config.json.example',
+        "list-pairs",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), True)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 7 active pairs: "
+            "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/USD, LTC/USDT, TKN/BTC.\n"
+            in captured.out)
+
+    # Test list-pairs subcommand with --all: all pairs
+    args = [
+        '--config', 'config.json.example',
+        "list-pairs", "--all",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), True)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 10 pairs: "
+            "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, LTC/USDT, NEO/BTC, "
+            "TKN/BTC, XRP/BTC.\n"
+            in captured.out)
+
+    # active markets, base=ETH, LTC
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--base", "ETH", "LTC",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 5 active markets with ETH, LTC as base currencies: "
+            "ETH/BTC, ETH/USDT, LTC/USD, LTC/USDT, XLTCUSDT.\n"
+            in captured.out)
+
+    # active markets, base=LTC
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--base", "LTC",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 3 active markets with LTC as base currency: "
+            "LTC/USD, LTC/USDT, XLTCUSDT.\n"
+            in captured.out)
+
+    # active markets, quote=USDT, USD
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--quote", "USDT", "USD",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 4 active markets with USDT, USD as quote currencies: "
+            "ETH/USDT, LTC/USD, LTC/USDT, XLTCUSDT.\n"
+            in captured.out)
+
+    # active markets, quote=USDT
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--quote", "USDT",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 3 active markets with USDT as quote currency: "
+            "ETH/USDT, LTC/USDT, XLTCUSDT.\n"
+            in captured.out)
+
+    # active markets, base=LTC, quote=USDT
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--base", "LTC", "--quote", "USDT",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 2 active markets with LTC as base currency and "
+            "with USDT as quote currency: LTC/USDT, XLTCUSDT.\n"
+            in captured.out)
+
+    # active pairs, base=LTC, quote=USDT
+    args = [
+        '--config', 'config.json.example',
+        "list-pairs",
+        "--base", "LTC", "--quote", "USDT",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), True)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 1 active pair with LTC as base currency and "
+            "with USDT as quote currency: LTC/USDT.\n"
+            in captured.out)
+
+    # active markets, base=LTC, quote=USDT, NONEXISTENT
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--base", "LTC", "--quote", "USDT", "NONEXISTENT",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 2 active markets with LTC as base currency and "
+            "with USDT, NONEXISTENT as quote currencies: LTC/USDT, XLTCUSDT.\n"
+            in captured.out)
+
+    # active markets, base=LTC, quote=NONEXISTENT
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--base", "LTC", "--quote", "NONEXISTENT",
+        "--print-list",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 0 active markets with LTC as base currency and "
+            "with NONEXISTENT as quote currency.\n"
+            in captured.out)
+
+    # Test tabular output
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 8 active markets:\n"
+            in captured.out)
+
+    # Test tabular output, no markets found
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--base", "LTC", "--quote", "NONEXISTENT",
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Exchange Bittrex has 0 active markets with LTC as base currency and "
+            "with NONEXISTENT as quote currency.\n"
+            in captured.out)
+
+    # Test --print-json
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--print-json"
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ('["BLK/BTC","BTT/BTC","ETH/BTC","ETH/USDT","LTC/USD","LTC/USDT","TKN/BTC","XLTCUSDT"]'
+            in captured.out)
+
+    # Test --print-csv
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--print-csv"
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert ("Id,Symbol,Base,Quote,Active,Is pair" in captured.out)
+    assert ("blkbtc,BLK/BTC,BLK,BTC,True,True" in captured.out)
+    assert ("BTTBTC,BTT/BTC,BTT,BTC,True,True" in captured.out)
+
+    # Test --one-column
+    args = [
+        '--config', 'config.json.example',
+        "list-markets",
+        "--one-column"
+    ]
+    start_list_markets(get_args(args), False)
+    captured = capsys.readouterr()
+    assert re.search(r"^BLK/BTC$", captured.out, re.MULTILINE)
+    assert re.search(r"^BTT/BTC$", captured.out, re.MULTILINE)
+
 
 def test_create_datadir_failed(caplog):
 
@@ -92,7 +449,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets):
 def test_download_data_no_markets(mocker, caplog):
     dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
                            MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
-    patch_exchange(mocker)
+    patch_exchange(mocker, id='binance')
     mocker.patch(
         'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
     )
@@ -104,7 +461,7 @@ def test_download_data_no_markets(mocker, caplog):
     ]
     start_download_data(get_args(args))
     assert dl_mock.call_args[1]['timerange'].starttype == "date"
-    assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)
+    assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange Binance.", caplog)
 
 
 def test_download_data_no_exchange(mocker, caplog):
@@ -144,3 +501,25 @@ def test_download_data_no_pairs(mocker, caplog):
     with pytest.raises(OperationalException,
                        match=r"Downloading data requires a list of pairs\..*"):
         start_download_data(pargs)
+
+
+def test_download_data_trades(mocker, caplog):
+    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_trades_data',
+                           MagicMock(return_value=[]))
+    convert_mock = mocker.patch('freqtrade.utils.convert_trades_to_ohlcv',
+                                MagicMock(return_value=[]))
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    args = [
+        "download-data",
+        "--exchange", "kraken",
+        "--pairs", "ETH/BTC", "XRP/BTC",
+        "--days", "20",
+        "--dl-trades"
+    ]
+    start_download_data(get_args(args))
+    assert dl_mock.call_args[1]['timerange'].starttype == "date"
+    assert dl_mock.call_count == 1
+    assert convert_mock.call_count == 1