Merge branch 'develop' into dataprovider-add-ticker

2020-05-14 13:22:52 +03:00 · 2020-05-14 13:22:52 +03:00 · 3079e18239
commit 3079e18239
parent 63b55658ac d957f3dd2d
53 changed files with 886 additions and 237 deletions
--- a/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl
+++ b/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl
--- a/build_helpers/TA_Lib-0.4.17-cp38-cp38-win_amd64.whl
+++ b/build_helpers/TA_Lib-0.4.17-cp38-cp38-win_amd64.whl
--- a/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl
+++ b/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl
--- a/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl
+++ b/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl
--- a/build_helpers/install_windows.ps1
+++ b/build_helpers/install_windows.ps1
@ -7,10 +7,10 @@ python -m pip install --upgrade pip
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"

 if ($pyv -eq '3.7') {
-    pip install build_helpers\TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl
 }
 if ($pyv -eq '3.8') {
-    pip install build_helpers\TA_Lib-0.4.17-cp38-cp38-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.18-cp38-cp38-win_amd64.whl
 }

 pip install -r requirements-dev.txt
--- a/config_full.json.example
+++ b/config_full.json.example
@ -120,6 +120,7 @@
        "enabled": false,
        "listen_ip_address": "127.0.0.1",
        "listen_port": 8080,
+        "jwt_secret_key": "somethingrandom",
        "username": "freqtrader",
        "password": "SuperSecurePassword"
    },
--- a/docs/configuration.md
+++ b/docs/configuration.md
@ -108,7 +108,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below. <br> **Datatype:** Boolean
 | `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`. <br> **Datatype:** ClassName
 | `strategy_path` | Adds an additional strategy lookup path (must be a directory). <br> **Datatype:** String
-| `internals.process_throttle_secs` | Set the process throttle. Value in second. <br>*Defaults to `5` seconds.* <br> **Datatype:** Positive Intege
+| `internals.process_throttle_secs` | Set the process throttle. Value in second. <br>*Defaults to `5` seconds.* <br> **Datatype:** Positive Integer
 | `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages. <br>*Defaults to `60` seconds.* <br> **Datatype:** Positive Integer or 0
 | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. <br> **Datatype:** Boolean
 | `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file. <br> **Datatype:** String
--- a/docs/docker.md
+++ b/docs/docker.md
@ -65,7 +65,7 @@ docker-compose up -d

 #### Docker-compose logs

-Logs will be written to `user_data/freqtrade.log`. 
+Logs will be written to `user_data/logs/freqtrade.log`. 
 Alternatively, you can check the latest logs using `docker-compose logs -f`.

 #### Database
--- a/docs/installation.md
+++ b/docs/installation.md
@ -248,14 +248,14 @@ git clone https://github.com/freqtrade/freqtrade.git

 Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).

-As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl` (make sure to use the version matching your python version)
+As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version)

 ```cmd
 >cd \path\freqtrade-develop
 >python -m venv .env
 >.env\Scripts\activate.bat
 REM optionally install ta-lib from wheel
-REM >pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl
+REM >pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl
 >pip install -r requirements.txt
 >pip install -e .
 >freqtrade
--- a/docs/requirements-docs.txt
+++ b/docs/requirements-docs.txt
@ -1,2 +1,2 @@
-mkdocs-material==5.1.3
+mkdocs-material==5.1.6
 mdx_truly_sane_lists==1.2
--- a/docs/rest-api.md
+++ b/docs/rest-api.md
@ -11,6 +11,7 @@ Sample configuration:
        "enabled": true,
        "listen_ip_address": "127.0.0.1",
        "listen_port": 8080,
+        "jwt_secret_key": "somethingrandom",
        "username": "Freqtrader",
        "password": "SuperSecret1!"
    },
@ -29,7 +30,7 @@ This should return the response:
 {"status":"pong"}
 ```

-All other endpoints return sensitive info and require authentication, so are not available through a web browser.
+All other endpoints return sensitive info and require authentication and are therefore not available through a web browser.

 To generate a secure password, either use a password manager, or use the below code snipped.

@ -38,6 +39,9 @@ import secrets
 secrets.token_hex()
 ```

+!!! Hint
+    Use the same method to also generate a JWT secret key (`jwt_secret_key`).
+
 ### Configuration with docker

 If you run your bot using docker, you'll need to have the bot listen to incomming connections. The security is then handled by docker.
@ -202,3 +206,28 @@ whitelist
        Show the current whitelist
        :returns: json object
 ```
+
+## Advanced API usage using JWT tokens
+
+!!! Note
+    The below should be done in an application (a Freqtrade REST API client, which fetches info via API), and is not intended to be used on a regular basis.
+
+Freqtrade's REST API also offers JWT (JSON Web Tokens).
+You can login using the following command, and subsequently use the resulting access_token.
+
+``` bash
+> curl -X POST --user Freqtrader http://localhost:8080/api/v1/token/login
+{"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiMmEwYmY0NWUtMjhmOS00YTUzLTlmNzItMmM5ZWVlYThkNzc2IiwiZXhwIjoxNTg5MTIwNTgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.qt6MAXYIa-l556OM7arBvYJ0SDI9J8bIk3_glDujF5g","refresh_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiZWQ1ZWI3YjAtYjMwMy00YzAyLTg2N2MtNWViMjIxNWQ2YTMxIiwiZXhwIjoxNTkxNzExNjgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJ0eXBlIjoicmVmcmVzaCJ9.d1AT_jYICyTAjD0fiQAr52rkRqtxCjUGEMwlNuuzgNQ"}
+
+> access_token="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiMmEwYmY0NWUtMjhmOS00YTUzLTlmNzItMmM5ZWVlYThkNzc2IiwiZXhwIjoxNTg5MTIwNTgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.qt6MAXYIa-l556OM7arBvYJ0SDI9J8bIk3_glDujF5g"
+# Use access_token for authentication
+> curl -X GET --header "Authorization: Bearer ${access_token}" http://localhost:8080/api/v1/count
+
+```
+
+Since the access token has a short timeout (15 min) - the `token/refresh` request should be used periodically to get a fresh access token:
+
+``` bash
+> curl -X POST --header "Authorization: Bearer ${refresh_token}"http://localhost:8080/api/v1/token/refresh
+{"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk5NzQsIm5iZiI6MTU4OTExOTk3NCwianRpIjoiMDBjNTlhMWUtMjBmYS00ZTk0LTliZjAtNWQwNTg2MTdiZDIyIiwiZXhwIjoxNTg5MTIwODc0LCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.1seHlII3WprjjclY6DpRhen0rqdF4j6jbvxIhUFaSbs"}
+```
--- a/docs/strategy-advanced.md
+++ b/docs/strategy-advanced.md
@ -20,7 +20,7 @@ It applies a tight timeout for higher priced assets, while allowing more time to
 The function must return either `True` (cancel order) or `False` (keep order alive).

 ``` python
-from datetime import datetime, timestamp
+from datetime import datetime, timedelta
 from freqtrade.persistence import Trade

 class Awesomestrategy(IStrategy):
@ -59,7 +59,7 @@ class Awesomestrategy(IStrategy):
 ### Custom order timeout example (using additional data)

 ``` python
-from datetime import datetime, timestamp
+from datetime import datetime
 from freqtrade.persistence import Trade

 class Awesomestrategy(IStrategy):
--- a/docs/strategy-customization.md
+++ b/docs/strategy-customization.md
@ -324,67 +324,14 @@ class Awesomestrategy(IStrategy):
 !!! Note
    If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.

-### Additional data (DataProvider)
+***

-The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy.
-
-All methods return `None` in case of failure (do not raise an exception).
-
-Please always check the mode of operation to select the correct method to get data (samples see below).
-
-#### Possible options for DataProvider
-
- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval).
- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame.
- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk.
- `get_pair_dataframe(pair, timeframe)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes).
- `orderbook(pair, maximum)` - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries.
- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure.
- `runmode` - Property containing the current runmode.
-
-#### Example: fetch live / historical candle (OHLCV) data for the first informative pair
-
-``` python
-if self.dp:
-    inf_pair, inf_timeframe = self.informative_pairs()[0]
-    informative = self.dp.get_pair_dataframe(pair=inf_pair,
-                                             timeframe=inf_timeframe)
-```
-
-!!! Warning "Warning about backtesting"
-    Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
-    for the backtesting runmode) provides the full time-range in one go,
-    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode).
-
-!!! Warning "Warning in hyperopt"
-    This option cannot currently be used during hyperopt.
-
-#### Orderbook
-
-``` python
-if self.dp:
-    if self.dp.runmode.value in ('live', 'dry_run'):
-        ob = self.dp.orderbook(metadata['pair'], 1)
-        dataframe['best_bid'] = ob['bids'][0][0]
-        dataframe['best_ask'] = ob['asks'][0][0]
-```
-
-!!! Warning
-    The order book is not part of the historic data which means backtesting and hyperopt will not work if this
-    method is used.
-
-#### Available Pairs
-
-``` python
-if self.dp:
-    for pair, timeframe in self.dp.available_pairs:
-        print(f"available {pair}, {timeframe}")
-```
+### Additional data (informative_pairs)

 #### Get data for non-tradeable pairs

 Data for additional, informative pairs (reference pairs) can be beneficial for some strategies.
-Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see above).
+Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below).
 These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting.

 The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument.
@ -404,6 +351,107 @@ def informative_pairs(self):
    It is however better to use resampling to longer time-intervals when possible
    to avoid hammering the exchange with too many requests and risk being blocked.

+***
+
+### Additional data (DataProvider)
+
+The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy.
+
+All methods return `None` in case of failure (do not raise an exception).
+
+Please always check the mode of operation to select the correct method to get data (samples see below).
+
+#### Possible options for DataProvider
+
+- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval).
+- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist)
+- [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes).
+- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk.
+- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure.
+- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame.
+- [`orderbook(pair, maximum)`](#orderbookpair-maximum) - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries.
+- `runmode` - Property containing the current runmode.
+
+#### Example Usages:
+
+#### *available_pairs*
+
+``` python
+if self.dp:
+    for pair, timeframe in self.dp.available_pairs:
+        print(f"available {pair}, {timeframe}")
+```
+
+#### *current_whitelist()*
+Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. 
+
+The strategy might look something like this:
+
+*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day ATR to buy and sell.*
+
+Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least!
+
+Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use.
+
+This is where calling `self.dp.current_whitelist()` comes in handy.
+
+```python
+class SampleStrategy(IStrategy):
+    # strategy init stuff...
+
+    ticker_interval = '5m'
+
+    # more strategy init stuff..
+
+    def informative_pairs(self):
+
+        # get access to all pairs available in whitelist. 
+        pairs = self.dp.current_whitelist()
+        # Assign tf to each pair so they can be downloaded and cached for strategy.
+        informative_pairs = [(pair, '1d') for pair in pairs]
+        return informative_pairs
+
+    def populate_indicators(self, dataframe, metadata):
+        # Get the informative pair
+        informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d')
+        # Get the 14 day ATR.
+        atr = ta.ATR(informative, timeperiod=14)
+        # Do other stuff
+```
+
+#### *get_pair_dataframe(pair, timeframe)*
+
+``` python
+# fetch live / historical candle (OHLCV) data for the first informative pair
+if self.dp:
+    inf_pair, inf_timeframe = self.informative_pairs()[0]
+    informative = self.dp.get_pair_dataframe(pair=inf_pair,
+                                             timeframe=inf_timeframe)
+```
+
+!!! Warning "Warning about backtesting"
+    Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
+    for the backtesting runmode) provides the full time-range in one go,
+    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode).
+
+!!! Warning "Warning in hyperopt"
+    This option cannot currently be used during hyperopt.
+
+#### *orderbook(pair, maximum)*
+
+``` python
+if self.dp:
+    if self.dp.runmode.value in ('live', 'dry_run'):
+        ob = self.dp.orderbook(metadata['pair'], 1)
+        dataframe['best_bid'] = ob['bids'][0][0]
+        dataframe['best_ask'] = ob['asks'][0][0]
+```
+
+!!! Warning
+    The order book is not part of the historic data which means backtesting and hyperopt will not work if this
+    method is used.
+
+***
 ### Additional data (Wallets)

 The strategy provides access to the `Wallets` object. This contains the current balances on the exchange.
@ -426,6 +474,7 @@ if self.wallets:
 - `get_used(asset)` - currently tied up balance (open orders)
 - `get_total(asset)` - total available balance - sum of the 2 above

+***
 ### Additional data (Trades)

 A history of Trades can be retrieved in the strategy by querying the database.
--- a/docs/utils.md
+++ b/docs/utils.md
@ -521,3 +521,48 @@ Prints JSON data with details for the last best epoch (i.e., the best of all epo
 ```
 freqtrade hyperopt-show --best -n -1 --print-json --no-header
 ```
+
+## Show trades
+
+Print selected (or all) trades from database to screen.
+
+```
+usage: freqtrade show-trades [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+                             [-d PATH] [--userdir PATH] [--db-url PATH]
+                             [--trade-ids TRADE_IDS [TRADE_IDS ...]]
+                             [--print-json]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --db-url PATH         Override trades database URL, this is useful in custom
+                        deployments (default: `sqlite:///tradesv3.sqlite` for
+                        Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for
+                        Dry Run).
+  --trade-ids TRADE_IDS [TRADE_IDS ...]
+                        Specify the list of trade ids.
+  --print-json          Print output in JSON format.
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+```
+
+### Examples
+
+Print trades with id 2 and 3 as json
+
+``` bash
+freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print-json
+```
--- a/freqtrade/commands/init.py
+++ b/freqtrade/commands/init.py
@ -19,7 +19,8 @@ from freqtrade.commands.list_commands import (start_list_exchanges,
                                              start_list_hyperopts,
                                              start_list_markets,
                                              start_list_strategies,
-                                              start_list_timeframes)
+                                              start_list_timeframes,
+                                              start_show_trades)
 from freqtrade.commands.optimize_commands import (start_backtesting,
                                                  start_edge, start_hyperopt)
 from freqtrade.commands.pairlist_commands import start_test_pairlist
--- a/freqtrade/commands/arguments.py
+++ b/freqtrade/commands/arguments.py
@ -64,6 +64,8 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
                    "trade_source", "ticker_interval"]

+ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
+
 ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
                      "hyperopt_list_min_trades", "hyperopt_list_max_trades",
                      "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time",
@ -78,7 +80,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
 NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
                    "list-markets", "list-pairs", "list-strategies",
                    "list-hyperopts", "hyperopt-list", "hyperopt-show",
-                    "plot-dataframe", "plot-profit"]
+                    "plot-dataframe", "plot-profit", "show-trades"]

 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"]

@ -163,7 +165,7 @@ class Arguments:
                                        start_list_markets, start_list_strategies,
                                        start_list_timeframes, start_new_config,
                                        start_new_hyperopt, start_new_strategy,
-                                        start_plot_dataframe, start_plot_profit,
+                                        start_plot_dataframe, start_plot_profit, start_show_trades,
                                        start_backtesting, start_hyperopt, start_edge,
                                        start_test_pairlist, start_trading)

@ -330,6 +332,15 @@ class Arguments:
        plot_profit_cmd.set_defaults(func=start_plot_profit)
        self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)

+        # Add show-trades subcommand
+        show_trades = subparsers.add_parser(
+            'show-trades',
+            help='Show trades.',
+            parents=[_common_parser],
+        )
+        show_trades.set_defaults(func=start_show_trades)
+        self._build_args(optionlist=ARGS_SHOW_TRADES, parser=show_trades)
+
        # Add hyperopt-list subcommand
        hyperopt_list_cmd = subparsers.add_parser(
            'hyperopt-list',
--- a/freqtrade/commands/cli_options.py
+++ b/freqtrade/commands/cli_options.py
@ -217,7 +217,7 @@ AVAILABLE_CLI_OPTIONS = {
    ),
    "print_json": Arg(
        '--print-json',
-        help='Print best result detailization in JSON format.',
+        help='Print output in JSON format.',
        action='store_true',
        default=False,
    ),
@ -425,6 +425,11 @@ AVAILABLE_CLI_OPTIONS = {
        choices=["DB", "file"],
        default="file",
    ),
+    "trade_ids": Arg(
+        '--trade-ids',
+        help='Specify the list of trade ids.',
+        nargs='+',
+    ),
    # hyperopt-list, hyperopt-show
    "hyperopt_list_profitable": Arg(
        '--profitable',
--- a/freqtrade/commands/list_commands.py
+++ b/freqtrade/commands/list_commands.py
@ -197,3 +197,30 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
                  args.get('list_pairs_print_json', False) or
                  args.get('print_csv', False)):
            print(f"{summary_str}.")
+
+
+def start_show_trades(args: Dict[str, Any]) -> None:
+    """
+    Show trades
+    """
+    from freqtrade.persistence import init, Trade
+    import json
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    if 'db_url' not in config:
+        raise OperationalException("--db-url is required for this command.")
+
+    logger.info(f'Using DB: "{config["db_url"]}"')
+    init(config['db_url'], clean_open_orders=False)
+    tfilter = []
+
+    if config.get('trade_ids'):
+        tfilter.append(Trade.id.in_(config['trade_ids']))
+
+    trades = Trade.get_trades(tfilter).all()
+    logger.info(f"Printing {len(trades)} Trades: ")
+    if config.get('print_json', False):
+        print(json.dumps([trade.to_json() for trade in trades], indent=4))
+    else:
+        for trade in trades:
+            print(trade)
--- a/freqtrade/commands/trade_commands.py
+++ b/freqtrade/commands/trade_commands.py
@ -18,6 +18,9 @@ def start_trading(args: Dict[str, Any]) -> int:
    try:
        worker = Worker(args)
        worker.run()
+    except Exception as e:
+        logger.error(str(e))
+        logger.exception("Fatal exception!")
    except KeyboardInterrupt:
        logger.info('SIGINT received, aborting ...')
    finally:
--- a/freqtrade/configuration/configuration.py
+++ b/freqtrade/configuration/configuration.py
@ -351,8 +351,12 @@ class Configuration:
        self._args_to_config(config, argname='indicators2',
                             logstring='Using indicators2: {}')

+        self._args_to_config(config, argname='trade_ids',
+                             logstring='Filtering on trade_ids: {}')
+
        self._args_to_config(config, argname='plot_limit',
                             logstring='Limiting plot to: {}')
+
        self._args_to_config(config, argname='trade_source',
                             logstring='Using trades from: {}')

--- a/freqtrade/constants.py
+++ b/freqtrade/constants.py
@ -24,6 +24,9 @@ AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
 DRY_RUN_WALLET = 1000
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons
 DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
+# Don't modify sequence of DEFAULT_TRADES_COLUMNS
+# it has wide consequences for stored trades files
+DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']

 USERPATH_HYPEROPTS = 'hyperopts'
 USERPATH_STRATEGIES = 'strategies'
--- a/freqtrade/data/converter.py
+++ b/freqtrade/data/converter.py
@ -1,14 +1,17 @@
 """
 Functions to convert data from one format to another
 """
+import itertools
 import logging
 from datetime import datetime, timezone
-from typing import Any, Dict
+from operator import itemgetter
+from typing import Any, Dict, List

 import pandas as pd
 from pandas import DataFrame, to_datetime

-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
+from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
+                                 DEFAULT_TRADES_COLUMNS)

 logger = logging.getLogger(__name__)

@ -154,7 +157,27 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
    return frame


-def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
+def trades_remove_duplicates(trades: List[List]) -> List[List]:
+    """
+    Removes duplicates from the trades list.
+    Uses itertools.groupby to avoid converting to pandas.
+    Tests show it as being pretty efficient on lists of 4M Lists.
+    :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns
+    :return: same format as above, but with duplicates removed
+    """
+    return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))]
+
+
+def trades_dict_to_list(trades: List[Dict]) -> List[List]:
+    """
+    Convert fetch_trades result into a List (to be more memory efficient).
+    :param trades: List of trades, as returned by ccxt.fetch_trades.
+    :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns
+    """
+    return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades]
+
+
+def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame:
    """
    Converts trades list to OHLCV list
    TODO: This should get a dedicated test
@ -164,9 +187,10 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
    """
    from freqtrade.exchange import timeframe_to_minutes
    timeframe_minutes = timeframe_to_minutes(timeframe)
-    df = pd.DataFrame(trades)
-    df['datetime'] = pd.to_datetime(df['datetime'])
-    df = df.set_index('datetime')
+    df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS)
+    df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms',
+                                     utc=True,)
+    df = df.set_index('timestamp')

    df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
    df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()
--- a/freqtrade/data/dataprovider.py
+++ b/freqtrade/data/dataprovider.py
@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple
 from pandas import DataFrame

 from freqtrade.data.history import load_pair_history
+from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.state import RunMode

@ -18,9 +19,10 @@ logger = logging.getLogger(__name__)

 class DataProvider:

-    def __init__(self, config: dict, exchange: Exchange) -> None:
+    def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None:
        self._config = config
        self._exchange = exchange
+        self._pairlists = pairlists

    def refresh(self,
                pairlist: List[Tuple[str, str]],
@ -115,3 +117,17 @@ class DataProvider:
        can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other".
        """
        return RunMode(self._config.get('runmode', RunMode.OTHER))
+
+    def current_whitelist(self) -> List[str]:
+        """
+        fetch latest available whitelist.
+
+        Useful when you have a large whitelist and need to call each pair as an informative pair.
+        As available pairs does not show whitelist until after informative pairs have been cached.
+        :return: list of pairs in whitelist
+        """
+
+        if self._pairlists:
+            return self._pairlists.whitelist
+        else:
+            raise OperationalException("Dataprovider was not initialized with a pairlist provider.")
--- a/freqtrade/data/history/history_utils.py
+++ b/freqtrade/data/history/history_utils.py
@ -9,10 +9,13 @@ from pandas import DataFrame

 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
-from freqtrade.data.converter import ohlcv_to_dataframe, trades_to_ohlcv
+from freqtrade.data.converter import (ohlcv_to_dataframe,
+                                      trades_remove_duplicates,
+                                      trades_to_ohlcv)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
+from freqtrade.misc import format_ms_time

 logger = logging.getLogger(__name__)

@ -257,27 +260,40 @@ def _download_trades_history(exchange: Exchange,
    """
    try:

-        since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None
+        since = timerange.startts * 1000 if \
+            (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
+                days=-30).float_timestamp) * 1000

        trades = data_handler.trades_load(pair)

-        from_id = trades[-1]['id'] if trades else None
+        # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
+        # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+        # DEFAULT_TRADES_COLUMNS: 1 -> id

-        logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
-        logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
+        from_id = trades[-1][1] if trades else None
+        if trades and since < trades[-1][0]:
+            # Reset since to the last available point
+            # - 5 seconds (to ensure we're getting all trades)
+            since = trades[-1][0] - (5 * 1000)
+            logger.info(f"Using last trade date -5s - Downloading trades for {pair} "
+                        f"since: {format_ms_time(since)}.")
+
+        logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}")
+        logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}")
+        logger.info(f"Current Amount of trades: {len(trades)}")

        # Default since_ms to 30 days if nothing is given
        new_trades = exchange.get_historic_trades(pair=pair,
-                                                  since=since if since else
-                                                  int(arrow.utcnow().shift(
-                                                      days=-30).float_timestamp) * 1000,
+                                                  since=since,
                                                  from_id=from_id,
                                                  )
        trades.extend(new_trades[1])
+        # Remove duplicates to make sure we're not storing data we don't need
+        trades = trades_remove_duplicates(trades)
        data_handler.trades_store(pair, data=trades)

-        logger.debug("New Start: %s", trades[0]['datetime'])
-        logger.debug("New End: %s", trades[-1]['datetime'])
+        logger.debug(f"New Start: {format_ms_time(trades[0][0])}")
+        logger.debug(f"New End: {format_ms_time(trades[-1][0])}")
        logger.info(f"New Amount of trades: {len(trades)}")
        return True

--- a/freqtrade/data/history/idatahandler.py
+++ b/freqtrade/data/history/idatahandler.py
@ -8,16 +8,20 @@ from abc import ABC, abstractclassmethod, abstractmethod
 from copy import deepcopy
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Dict, List, Optional, Type
+from typing import List, Optional, Type

 from pandas import DataFrame

 from freqtrade.configuration import TimeRange
-from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
+from freqtrade.data.converter import (clean_ohlcv_dataframe,
+                                      trades_remove_duplicates, trim_dataframe)
 from freqtrade.exchange import timeframe_to_seconds

 logger = logging.getLogger(__name__)

+# Type for trades list
+TradeList = List[List]
+

 class IDataHandler(ABC):

@ -89,23 +93,25 @@ class IDataHandler(ABC):
        """

    @abstractmethod
-    def trades_store(self, pair: str, data: List[Dict]) -> None:
+    def trades_store(self, pair: str, data: TradeList) -> None:
        """
        Store trades data (list of Dicts) to file
        :param pair: Pair - used for filename
-        :param data: List of Dicts containing trade data
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
        """

    @abstractmethod
-    def trades_append(self, pair: str, data: List[Dict]):
+    def trades_append(self, pair: str, data: TradeList):
        """
        Append data to existing files
        :param pair: Pair - used for filename
-        :param data: List of Dicts containing trade data
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
        """

    @abstractmethod
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
        """
        Load a pair from file, either .json.gz or .json
        :param pair: Load trades for this pair
@ -121,6 +127,16 @@ class IDataHandler(ABC):
        :return: True when deleted, false if file did not exist.
        """

+    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+        """
+        Load a pair from file, either .json.gz or .json
+        Removes duplicates in the process.
+        :param pair: Load trades for this pair
+        :param timerange: Timerange to load trades for - currently not implemented
+        :return: List of trades
+        """
+        return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
+
    def ohlcv_load(self, pair, timeframe: str,
                   timerange: Optional[TimeRange] = None,
                   fill_missing: bool = True,
--- a/freqtrade/data/history/jsondatahandler.py
+++ b/freqtrade/data/history/jsondatahandler.py
@ -1,6 +1,7 @@
+import logging
 import re
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import List, Optional

 import numpy as np
 from pandas import DataFrame, read_json, to_datetime
@ -8,8 +9,11 @@ from pandas import DataFrame, read_json, to_datetime
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
+from freqtrade.data.converter import trades_dict_to_list

-from .idatahandler import IDataHandler
+from .idatahandler import IDataHandler, TradeList
+
+logger = logging.getLogger(__name__)


 class JsonDataHandler(IDataHandler):
@ -113,24 +117,26 @@ class JsonDataHandler(IDataHandler):
        # Check if regex found something and only return these results to avoid exceptions.
        return [match[0].replace('_', '/') for match in _tmp if match]

-    def trades_store(self, pair: str, data: List[Dict]) -> None:
+    def trades_store(self, pair: str, data: TradeList) -> None:
        """
        Store trades data (list of Dicts) to file
        :param pair: Pair - used for filename
-        :param data: List of Dicts containing trade data
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
        """
        filename = self._pair_trades_filename(self._datadir, pair)
        misc.file_dump_json(filename, data, is_zip=self._use_zip)

-    def trades_append(self, pair: str, data: List[Dict]):
+    def trades_append(self, pair: str, data: TradeList):
        """
        Append data to existing files
        :param pair: Pair - used for filename
-        :param data: List of Dicts containing trade data
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
        """
        raise NotImplementedError()

-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
        """
        Load a pair from file, either .json.gz or .json
        # TODO: respect timerange ...
@ -140,9 +146,15 @@ class JsonDataHandler(IDataHandler):
        """
        filename = self._pair_trades_filename(self._datadir, pair)
        tradesdata = misc.file_load_json(filename)
+
        if not tradesdata:
            return []

+        if isinstance(tradesdata[0], dict):
+            # Convert trades dict to list
+            logger.info("Old trades format detected - converting")
+            tradesdata = trades_dict_to_list(tradesdata)
+            pass
        return tradesdata

    def trades_purge(self, pair: str) -> bool:
--- a/freqtrade/edge/edge_positioning.py
+++ b/freqtrade/edge/edge_positioning.py
@ -238,20 +238,9 @@ class Edge:
        :param result Dataframe
        :return: result Dataframe
        """
-
-        # stake and fees
-        # stake = 0.015
-        # 0.05% is 0.0005
-        # fee = 0.001
-
-        # we set stake amount to an arbitrary amount.
-        # as it doesn't change the calculation.
-        # all returned values are relative.
-        # they are defined as ratios.
+        # We set stake amount to an arbitrary amount, as it doesn't change the calculation.
+        # All returned values are relative, they are defined as ratios.
        stake = 0.015
-        fee = self.fee
-        open_fee = fee / 2
-        close_fee = fee / 2

        result['trade_duration'] = result['close_time'] - result['open_time']

@ -262,12 +251,12 @@ class Edge:

        # Buy Price
        result['buy_vol'] = stake / result['open_rate']  # How many target are we buying
-        result['buy_fee'] = stake * open_fee
+        result['buy_fee'] = stake * self.fee
        result['buy_spend'] = stake + result['buy_fee']  # How much we're spending

        # Sell price
        result['sell_sum'] = result['buy_vol'] * result['close_rate']
-        result['sell_fee'] = result['sell_sum'] * close_fee
+        result['sell_fee'] = result['sell_sum'] * self.fee
        result['sell_take'] = result['sell_sum'] - result['sell_fee']

        # profit_ratio
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@ -18,13 +18,12 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE,
                                            TRUNCATE, decimal_to_precision)
 from pandas import DataFrame

-from freqtrade.data.converter import ohlcv_to_dataframe
+from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
 from freqtrade.exceptions import (DependencyException, InvalidOrderException,
                                  OperationalException, TemporaryError)
 from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
 from freqtrade.misc import deep_merge_dicts

-
 CcxtModuleType = Any


@ -769,7 +768,7 @@ class Exchange:
    @retrier_async
    async def _async_fetch_trades(self, pair: str,
                                  since: Optional[int] = None,
-                                  params: Optional[dict] = None) -> List[Dict]:
+                                  params: Optional[dict] = None) -> List[List]:
        """
        Asyncronously gets trade history using fetch_trades.
        Handles exchange errors, does one call to the exchange.
@ -789,7 +788,7 @@ class Exchange:
                    '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else ''
                )
                trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
-            return trades
+            return trades_dict_to_list(trades)
        except ccxt.NotSupported as e:
            raise OperationalException(
                f'Exchange {self._api.name} does not support fetching historical trade data.'
@ -803,7 +802,7 @@ class Exchange:
    async def _async_get_trade_history_id(self, pair: str,
                                          until: int,
                                          since: Optional[int] = None,
-                                          from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
+                                          from_id: Optional[str] = None) -> Tuple[str, List[List]]:
        """
        Asyncronously gets trade history using fetch_trades
        use this when exchange uses id-based iteration (check `self._trades_pagination`)
@ -814,7 +813,7 @@ class Exchange:
        returns tuple: (pair, trades-list)
        """

-        trades: List[Dict] = []
+        trades: List[List] = []

        if not from_id:
            # Fetch first elements using timebased method to get an ID to paginate on
@ -823,7 +822,9 @@ class Exchange:
            # e.g. Binance returns the "last 1000" candles within a 1h time interval
            # - so we will miss the first trades.
            t = await self._async_fetch_trades(pair, since=since)
-            from_id = t[-1]['id']
+            # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+            # DEFAULT_TRADES_COLUMNS: 1 -> id
+            from_id = t[-1][1]
            trades.extend(t[:-1])
        while True:
            t = await self._async_fetch_trades(pair,
@ -831,21 +832,21 @@ class Exchange:
            if len(t):
                # Skip last id since its the key for the next call
                trades.extend(t[:-1])
-                if from_id == t[-1]['id'] or t[-1]['timestamp'] > until:
+                if from_id == t[-1][1] or t[-1][0] > until:
                    logger.debug(f"Stopping because from_id did not change. "
-                                 f"Reached {t[-1]['timestamp']} > {until}")
+                                 f"Reached {t[-1][0]} > {until}")
                    # Reached the end of the defined-download period - add last trade as well.
                    trades.extend(t[-1:])
                    break

-                from_id = t[-1]['id']
+                from_id = t[-1][1]
            else:
                break

        return (pair, trades)

    async def _async_get_trade_history_time(self, pair: str, until: int,
-                                            since: Optional[int] = None) -> Tuple[str, List]:
+                                            since: Optional[int] = None) -> Tuple[str, List[List]]:
        """
        Asyncronously gets trade history using fetch_trades,
        when the exchange uses time-based iteration (check `self._trades_pagination`)
@ -855,16 +856,18 @@ class Exchange:
        returns tuple: (pair, trades-list)
        """

-        trades: List[Dict] = []
+        trades: List[List] = []
+        # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+        # DEFAULT_TRADES_COLUMNS: 1 -> id
        while True:
            t = await self._async_fetch_trades(pair, since=since)
            if len(t):
-                since = t[-1]['timestamp']
+                since = t[-1][1]
                trades.extend(t)
                # Reached the end of the defined-download period
-                if until and t[-1]['timestamp'] > until:
+                if until and t[-1][0] > until:
                    logger.debug(
-                        f"Stopping because until was reached. {t[-1]['timestamp']} > {until}")
+                        f"Stopping because until was reached. {t[-1][0]} > {until}")
                    break
            else:
                break
@ -874,7 +877,7 @@ class Exchange:
    async def _async_get_trade_history(self, pair: str,
                                       since: Optional[int] = None,
                                       until: Optional[int] = None,
-                                       from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
+                                       from_id: Optional[str] = None) -> Tuple[str, List[List]]:
        """
        Async wrapper handling downloading trades using either time or id based methods.
        """
@ -1041,9 +1044,9 @@ class Exchange:

            return matched_trades

-        except ccxt.NetworkError as e:
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
            raise TemporaryError(
-                f'Could not get trades due to networking error. Message: {e}') from e
+                f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e
        except ccxt.BaseError as e:
            raise OperationalException(e) from e

--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@ -54,8 +54,11 @@ class FreqtradeBot:
        # Init objects
        self.config = config

-        self._sell_rate_cache = TTLCache(maxsize=100, ttl=5)
-        self._buy_rate_cache = TTLCache(maxsize=100, ttl=5)
+        # Cache values for 1800 to avoid frequent polling of the exchange for prices
+        # Caching only applies to RPC methods, so prices for open trades are still
+        # refreshed once every iteration.
+        self._sell_rate_cache = TTLCache(maxsize=100, ttl=1800)
+        self._buy_rate_cache = TTLCache(maxsize=100, ttl=1800)

        self.strategy: IStrategy = StrategyResolver.load_strategy(self.config)

@ -68,15 +71,15 @@ class FreqtradeBot:

        self.wallets = Wallets(self.config, self.exchange)

-        self.dataprovider = DataProvider(self.config, self.exchange)
+        self.pairlists = PairListManager(self.exchange, self.config)
+
+        self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists)

        # Attach Dataprovider to Strategy baseclass
        IStrategy.dp = self.dataprovider
        # Attach Wallets to Strategy baseclass
        IStrategy.wallets = self.wallets

-        self.pairlists = PairListManager(self.exchange, self.config)
-
        # Initializing Edge only if enabled
        self.edge = Edge(self.config, self.exchange, self.strategy) if \
            self.config.get('edge', {}).get('enabled', False) else None
@ -898,7 +901,8 @@ class FreqtradeBot:
        Buy timeout - cancel order
        :return: True if order was fully cancelled
        """
-        if order['status'] != 'canceled':
+        # Cancelled orders may have the status of 'canceled' or 'closed'
+        if order['status'] not in ('canceled', 'closed'):
            reason = "cancelled due to timeout"
            corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
                                                            trade.amount)
@ -909,7 +913,10 @@ class FreqtradeBot:

        logger.info('Buy order %s for %s.', reason, trade)

-        if safe_value_fallback(corder, order, 'remaining', 'remaining') == order['amount']:
+        # Using filled to determine the filled amount
+        filled_amount = safe_value_fallback(corder, order, 'filled', 'filled')
+
+        if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
            logger.info('Buy order fully cancelled. Removing %s from database.', trade)
            # if trade is not partially completed, just delete the trade
            Trade.session.delete(trade)
@ -921,8 +928,7 @@ class FreqtradeBot:
        # cancel_order may not contain the full order dict, so we need to fallback
        # to the order dict aquired before cancelling.
        # we need to fall back to the values from order if corder does not contain these keys.
-        trade.amount = order['amount'] - safe_value_fallback(corder, order,
-                                                             'remaining', 'remaining')
+        trade.amount = filled_amount
        trade.stake_amount = trade.amount * trade.open_rate
        self.update_trade_state(trade, corder, trade.amount)

@ -943,8 +949,12 @@ class FreqtradeBot:
        if order['remaining'] == order['amount'] or order.get('filled') == 0.0:
            if not self.exchange.check_order_canceled_empty(order):
                reason = "cancelled due to timeout"
-                # if trade is not partially completed, just delete the trade
-                self.exchange.cancel_order(trade.open_order_id, trade.pair)
+                try:
+                    # if trade is not partially completed, just delete the trade
+                    self.exchange.cancel_order(trade.open_order_id, trade.pair)
+                except InvalidOrderException:
+                    logger.exception(f"Could not cancel sell order {trade.open_order_id}")
+                    return 'error cancelling order'
                logger.info('Sell order %s for %s.', reason, trade)
            else:
                reason = "cancelled on exchange"
@ -982,7 +992,7 @@ class FreqtradeBot:
        if wallet_amount >= amount:
            return amount
        elif wallet_amount > amount * 0.98:
-            logger.info(f"{pair} - Falling back to wallet-amount.")
+            logger.info(f"{pair} - Falling back to wallet-amount {wallet_amount} -> {amount}.")
            return wallet_amount
        else:
            raise DependencyException(
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@ -387,12 +387,19 @@ class Hyperopt:
        trials = json_normalize(results, max_level=1)
        trials['Best'] = ''
        trials['Stake currency'] = config['stake_currency']
-        trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
-                         'results_metrics.avg_profit', 'results_metrics.total_profit',
-                         'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
-                         'loss', 'is_initial_point', 'is_best']]
-        trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency',
-                          'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
+
+        base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
+                        'results_metrics.avg_profit', 'results_metrics.total_profit',
+                        'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
+                        'loss', 'is_initial_point', 'is_best']
+        param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
+        trials = trials[base_metrics + param_metrics]
+
+        base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency',
+                        'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
+        param_columns = list(results[0]['params_dict'].keys())
+        trials.columns = base_columns + param_columns
+
        trials['is_profit'] = False
        trials.loc[trials['is_initial_point'], 'Best'] = '*'
        trials.loc[trials['is_best'], 'Best'] = 'Best'
@ -648,7 +655,7 @@ class Hyperopt:
                        ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
                    ]
                with progressbar.ProgressBar(
-                         maxval=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
+                         max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
                         widgets=widgets
                     ) as pbar:
                    EVALS = ceil(self.total_epochs / jobs)
--- a/freqtrade/rpc/api_server.py
+++ b/freqtrade/rpc/api_server.py
@ -2,11 +2,16 @@ import logging
 import threading
 from datetime import date, datetime
 from ipaddress import IPv4Address
-from typing import Dict, Callable, Any
+from typing import Any, Callable, Dict

 from arrow import Arrow
 from flask import Flask, jsonify, request
 from flask.json import JSONEncoder
+from flask_jwt_extended import (JWTManager, create_access_token,
+                                create_refresh_token, get_jwt_identity,
+                                jwt_refresh_token_required,
+                                verify_jwt_in_request_optional)
+from werkzeug.security import safe_str_cmp
 from werkzeug.serving import make_server

 from freqtrade.__init__ import __version__
@ -38,9 +43,9 @@ class ArrowJSONEncoder(JSONEncoder):
 def require_login(func: Callable[[Any, Any], Any]):

    def func_wrapper(obj, *args, **kwargs):
-
+        verify_jwt_in_request_optional()
        auth = request.authorization
-        if auth and obj.check_auth(auth.username, auth.password):
+        if get_jwt_identity() or auth and obj.check_auth(auth.username, auth.password):
            return func(obj, *args, **kwargs)
        else:
            return jsonify({"error": "Unauthorized"}), 401
@ -70,8 +75,8 @@ class ApiServer(RPC):
    """

    def check_auth(self, username, password):
-        return (username == self._config['api_server'].get('username') and
-                password == self._config['api_server'].get('password'))
+        return (safe_str_cmp(username, self._config['api_server'].get('username')) and
+                safe_str_cmp(password, self._config['api_server'].get('password')))

    def __init__(self, freqtrade) -> None:
        """
@ -83,6 +88,12 @@ class ApiServer(RPC):

        self._config = freqtrade.config
        self.app = Flask(__name__)
+
+        # Setup the Flask-JWT-Extended extension
+        self.app.config['JWT_SECRET_KEY'] = self._config['api_server'].get(
+            'jwt_secret_key', 'super-secret')
+
+        self.jwt = JWTManager(self.app)
        self.app.json_encoder = ArrowJSONEncoder

        # Register application handling
@ -148,6 +159,10 @@ class ApiServer(RPC):
        self.app.register_error_handler(404, self.page_not_found)

        # Actions to control the bot
+        self.app.add_url_rule(f'{BASE_URI}/token/login', 'login',
+                              view_func=self._token_login, methods=['POST'])
+        self.app.add_url_rule(f'{BASE_URI}/token/refresh', 'token_refresh',
+                              view_func=self._token_refresh, methods=['POST'])
        self.app.add_url_rule(f'{BASE_URI}/start', 'start',
                              view_func=self._start, methods=['POST'])
        self.app.add_url_rule(f'{BASE_URI}/stop', 'stop', view_func=self._stop, methods=['POST'])
@ -199,6 +214,37 @@ class ApiServer(RPC):
            'code': 404
        }), 404

+    @require_login
+    @rpc_catch_errors
+    def _token_login(self):
+        """
+        Handler for /token/login
+        Returns a JWT token
+        """
+        auth = request.authorization
+        if auth and self.check_auth(auth.username, auth.password):
+            keystuff = {'u': auth.username}
+            ret = {
+                'access_token': create_access_token(identity=keystuff),
+                'refresh_token': create_refresh_token(identity=keystuff),
+            }
+            return self.rest_dump(ret)
+
+        return jsonify({"error": "Unauthorized"}), 401
+
+    @jwt_refresh_token_required
+    @rpc_catch_errors
+    def _token_refresh(self):
+        """
+        Handler for /token/refresh
+        Returns a JWT token based on a JWT refresh token
+        """
+        current_user = get_jwt_identity()
+        new_token = create_access_token(identity=current_user, fresh=False)
+
+        ret = {'access_token': new_token}
+        return self.rest_dump(ret)
+
    @require_login
    @rpc_catch_errors
    def _start(self):
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@ -94,6 +94,7 @@ class RPC:
            'dry_run': config['dry_run'],
            'stake_currency': config['stake_currency'],
            'stake_amount': config['stake_amount'],
+            'max_open_trades': config['max_open_trades'],
            'minimal_roi': config['minimal_roi'].copy(),
            'stoploss': config['stoploss'],
            'trailing_stop': config['trailing_stop'],
@ -103,6 +104,8 @@ class RPC:
            'ticker_interval': config['ticker_interval'],
            'exchange': config['exchange']['name'],
            'strategy': config['strategy'],
+            'forcebuy_enabled': config.get('forcebuy_enable', False),
+            'state': str(self._freqtrade.state)
        }
        return val

--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@ -579,7 +579,7 @@ class Telegram(RPC):
                  "*/whitelist:* `Show current whitelist` \n" \
                  "*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs " \
                  "to the blacklist.` \n" \
-                  "*/edge:* `Shows validated pairs by Edge if it is enabeld` \n" \
+                  "*/edge:* `Shows validated pairs by Edge if it is enabled` \n" \
                  "*/help:* `This help message`\n" \
                  "*/version:* `Show version`"

@ -621,10 +621,12 @@ class Telegram(RPC):
            f"*Mode:* `{'Dry-run' if val['dry_run'] else 'Live'}`\n"
            f"*Exchange:* `{val['exchange']}`\n"
            f"*Stake per trade:* `{val['stake_amount']} {val['stake_currency']}`\n"
+            f"*Max open Trades:* `{val['max_open_trades']}`\n"
            f"*Minimum ROI:* `{val['minimal_roi']}`\n"
            f"{sl_info}"
            f"*Ticker Interval:* `{val['ticker_interval']}`\n"
-            f"*Strategy:* `{val['strategy']}`"
+            f"*Strategy:* `{val['strategy']}`\n"
+            f"*Current state:* `{val['state']}`"
        )

    def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None:
--- a/freqtrade/state.py
+++ b/freqtrade/state.py
@ -14,6 +14,9 @@ class State(Enum):
    STOPPED = 2
    RELOAD_CONF = 3

+    def __str__(self):
+        return f"{self.name.lower()}"
+

 class RunMode(Enum):
    """
--- a/requirements-common.txt
+++ b/requirements-common.txt
@ -1,15 +1,15 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.27.1
+ccxt==1.27.49
 SQLAlchemy==1.3.16
-python-telegram-bot==12.6.1
-arrow==0.15.5
+python-telegram-bot==12.7
+arrow==0.15.6
 cachetools==4.1.0
 requests==2.23.0
 urllib3==1.25.9
 wrapt==1.12.1
 jsonschema==3.2.0
-TA-Lib==0.4.17
+TA-Lib==0.4.18
 tabulate==0.8.7
 pycoingecko==1.2.0
 jinja2==2.11.2
@ -25,6 +25,7 @@ sdnotify==0.3.2

 # Api server
 flask==1.1.2
+flask-jwt-extended==3.24.1

 # Support for colorized terminal output
 colorama==0.4.3
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@ -8,8 +8,8 @@ flake8==3.7.9
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==4.1.0
 mypy==0.770
-pytest==5.4.1
-pytest-asyncio==0.11.0
+pytest==5.4.2
+pytest-asyncio==0.12.0
 pytest-cov==2.8.1
 pytest-mock==3.1.0
 pytest-random-order==1.0.4
--- a/requirements-hyperopt.txt
+++ b/requirements-hyperopt.txt
@ -7,4 +7,4 @@ scikit-learn==0.22.2.post1
 scikit-optimize==0.7.4
 filelock==3.0.12
 joblib==0.14.1
-progressbar2==3.51.0
+progressbar2==3.51.3
--- a/requirements-plot.txt
+++ b/requirements-plot.txt
@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt

-plotly==4.6.0
+plotly==4.7.1

--- a/requirements.txt
+++ b/requirements.txt
@ -1,5 +1,5 @@
 # Load common requirements
 -r requirements-common.txt

-numpy==1.18.3
+numpy==1.18.4
 pandas==1.0.3
--- a/setup.py
+++ b/setup.py
@ -16,7 +16,7 @@ if readme_file.is_file():
    readme_long = (Path(__file__).parent / "README.md").read_text()

 # Requirements used for submodules
-api = ['flask']
+api = ['flask', 'flask-jwt-extended']
 plot = ['plotly>=4.0']
 hyperopt = [
    'scipy',
--- a/tests/commands/test_commands.py
+++ b/tests/commands/test_commands.py
@ -10,11 +10,13 @@ from freqtrade.commands import (start_convert_data, start_create_userdir,
                                start_list_hyperopts, start_list_markets,
                                start_list_strategies, start_list_timeframes,
                                start_new_hyperopt, start_new_strategy,
-                                start_test_pairlist, start_trading)
+                                start_show_trades, start_test_pairlist,
+                                start_trading)
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.exceptions import OperationalException
 from freqtrade.state import RunMode
-from tests.conftest import (get_args, log_has, log_has_re, patch_exchange,
+from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re,
+                            patch_exchange,
                            patched_configuration_load_config_file)


@ -30,7 +32,7 @@ def test_setup_utils_configuration():
    assert config['exchange']['secret'] == ''


-def test_start_trading_fail(mocker):
+def test_start_trading_fail(mocker, caplog):

    mocker.patch("freqtrade.worker.Worker.run", MagicMock(side_effect=OperationalException))

@ -41,16 +43,15 @@ def test_start_trading_fail(mocker):
        'trade',
        '-c', 'config.json.example'
    ]
-    with pytest.raises(OperationalException):
-        start_trading(get_args(args))
+    start_trading(get_args(args))
    assert exitmock.call_count == 1

    exitmock.reset_mock()
-
+    caplog.clear()
    mocker.patch("freqtrade.worker.Worker.__init__", MagicMock(side_effect=OperationalException))
-    with pytest.raises(OperationalException):
-        start_trading(get_args(args))
+    start_trading(get_args(args))
    assert exitmock.call_count == 0
+    assert log_has('Fatal exception!', caplog)


 def test_list_exchanges(capsys):
@ -1040,3 +1041,46 @@ def test_convert_data_trades(mocker, testdatadir):
    assert trades_mock.call_args[1]['convert_from'] == 'jsongz'
    assert trades_mock.call_args[1]['convert_to'] == 'json'
    assert trades_mock.call_args[1]['erase'] is False
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_show_trades(mocker, fee, capsys, caplog):
+    mocker.patch("freqtrade.persistence.init")
+    create_mock_trades(fee)
+    args = [
+        "show-trades",
+        "--db-url",
+        "sqlite:///"
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_show_trades(pargs)
+    assert log_has("Printing 3 Trades: ", caplog)
+    captured = capsys.readouterr()
+    assert "Trade(id=1" in captured.out
+    assert "Trade(id=2" in captured.out
+    assert "Trade(id=3" in captured.out
+    args = [
+        "show-trades",
+        "--db-url",
+        "sqlite:///",
+        "--print-json",
+        "--trade-ids", "1", "2"
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_show_trades(pargs)
+
+    captured = capsys.readouterr()
+    assert log_has("Printing 2 Trades: ", caplog)
+    assert '"trade_id": 1' in captured.out
+    assert '"trade_id": 2' in captured.out
+    assert '"trade_id": 3' not in captured.out
+    args = [
+        "show-trades",
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+
+    with pytest.raises(OperationalException, match=r"--db-url is required for this command."):
+        start_show_trades(pargs)
--- a/tests/conftest.py
+++ b/tests/conftest.py
@ -304,7 +304,8 @@ def default_conf(testdatadir):
        "user_data_dir": Path("user_data"),
        "verbosity": 3,
        "strategy_path": str(Path(__file__).parent / "strategy" / "strats"),
-        "strategy": "DefaultStrategy"
+        "strategy": "DefaultStrategy",
+        "internals": {},
    }
    return configuration

@ -877,6 +878,99 @@ def limit_buy_order_old_partial_canceled(limit_buy_order_old_partial):
    return res


+@pytest.fixture(scope='function')
+def limit_buy_order_canceled_empty(request):
+    # Indirect fixture
+    # Documentation:
+    # https://docs.pytest.org/en/latest/example/parametrize.html#apply-indirect-on-particular-arguments
+
+    exchange_name = request.param
+    if exchange_name == 'ftx':
+        return {
+            'info': {},
+            'id': '1234512345',
+            'clientOrderId': None,
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 34.3225,
+            'amount': 0.55,
+            'cost': 0.0,
+            'average': None,
+            'filled': 0.0,
+            'remaining': 0.0,
+            'status': 'closed',
+            'fee': None,
+            'trades': None
+        }
+    elif exchange_name == 'kraken':
+        return {
+            'info': {},
+            'id': 'AZNPFF-4AC4N-7MKTAT',
+            'clientOrderId': None,
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'status': 'canceled',
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 34.3225,
+            'cost': 0.0,
+            'amount': 0.55,
+            'filled': 0.0,
+            'average': 0.0,
+            'remaining': 0.55,
+            'fee': {'cost': 0.0, 'rate': None, 'currency': 'USDT'},
+            'trades': []
+        }
+    elif exchange_name == 'binance':
+        return {
+            'info': {},
+            'id': '1234512345',
+            'clientOrderId': 'alb1234123',
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 0.016804,
+            'amount': 0.55,
+            'cost': 0.0,
+            'average': None,
+            'filled': 0.0,
+            'remaining': 0.55,
+            'status': 'canceled',
+            'fee': None,
+            'trades': None
+        }
+    else:
+        return {
+            'info': {},
+            'id': '1234512345',
+            'clientOrderId': 'alb1234123',
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 0.016804,
+            'amount': 0.55,
+            'cost': 0.0,
+            'average': None,
+            'filled': 0.0,
+            'remaining': 0.55,
+            'status': 'canceled',
+            'fee': None,
+            'trades': None
+        }
+
+
@pytest.fixture
 def limit_sell_order():
    return {
@ -1328,6 +1422,15 @@ def trades_for_order():

@pytest.fixture(scope="function")
 def trades_history():
+    return [[1565798399463, '126181329', None, 'buy', 0.019627, 0.04, 0.00078508],
+            [1565798399629, '126181330', None, 'buy', 0.019627, 0.244, 0.004788987999999999],
+            [1565798399752, '126181331', None, 'sell', 0.019626, 0.011, 0.00021588599999999999],
+            [1565798399862, '126181332', None, 'sell', 0.019626, 0.011, 0.00021588599999999999],
+            [1565798399872, '126181333', None, 'sell', 0.019626, 0.011, 0.00021588599999999999]]
+
+
+@pytest.fixture(scope="function")
+def fetch_trades_result():
    return [{'info': {'a': 126181329,
                      'p': '0.01962700',
                      'q': '0.04000000',
--- a/tests/data/test_converter.py
+++ b/tests/data/test_converter.py
@ -5,12 +5,10 @@ from freqtrade.configuration.timerange import TimeRange
 from freqtrade.data.converter import (convert_ohlcv_format,
                                      convert_trades_format,
                                      ohlcv_fill_up_missing_data,
-                                      ohlcv_to_dataframe,
-                                      trim_dataframe)
-from freqtrade.data.history import (get_timerange,
-                                    load_data,
-                                    load_pair_history,
-                                    validate_backtest_data)
+                                      ohlcv_to_dataframe, trades_dict_to_list,
+                                      trades_remove_duplicates, trim_dataframe)
+from freqtrade.data.history import (get_timerange, load_data,
+                                    load_pair_history, validate_backtest_data)
 from tests.conftest import log_has
 from tests.data.test_history import _backup_file, _clean_test_file

@ -197,32 +195,60 @@ def test_trim_dataframe(testdatadir) -> None:
    assert all(data_modify.iloc[0] == data.iloc[25])


-def test_convert_trades_format(mocker, default_conf, testdatadir):
-    file = testdatadir / "XRP_ETH-trades.json.gz"
-    file_new = testdatadir / "XRP_ETH-trades.json"
-    _backup_file(file, copy_file=True)
-    default_conf['datadir'] = testdatadir
+def test_trades_remove_duplicates(trades_history):
+    trades_history1 = trades_history * 3
+    assert len(trades_history1) == len(trades_history) * 3
+    res = trades_remove_duplicates(trades_history1)
+    assert len(res) == len(trades_history)
+    for i, t in enumerate(res):
+        assert t == trades_history[i]

-    assert not file_new.exists()
+
+def test_trades_dict_to_list(fetch_trades_result):
+    res = trades_dict_to_list(fetch_trades_result)
+    assert isinstance(res, list)
+    assert isinstance(res[0], list)
+    for i, t in enumerate(res):
+        assert t[0] == fetch_trades_result[i]['timestamp']
+        assert t[1] == fetch_trades_result[i]['id']
+        assert t[2] == fetch_trades_result[i]['type']
+        assert t[3] == fetch_trades_result[i]['side']
+        assert t[4] == fetch_trades_result[i]['price']
+        assert t[5] == fetch_trades_result[i]['amount']
+        assert t[6] == fetch_trades_result[i]['cost']
+
+
+def test_convert_trades_format(mocker, default_conf, testdatadir):
+    files = [{'old': testdatadir / "XRP_ETH-trades.json.gz",
+              'new': testdatadir / "XRP_ETH-trades.json"},
+             {'old': testdatadir / "XRP_OLD-trades.json.gz",
+              'new': testdatadir / "XRP_OLD-trades.json"},
+             ]
+    for file in files:
+        _backup_file(file['old'], copy_file=True)
+        assert not file['new'].exists()
+
+    default_conf['datadir'] = testdatadir

    convert_trades_format(default_conf, convert_from='jsongz',
                          convert_to='json', erase=False)

-    assert file_new.exists()
-    assert file.exists()
+    for file in files:
+        assert file['new'].exists()
+        assert file['old'].exists()

-    # Remove original file
-    file.unlink()
+        # Remove original file
+        file['old'].unlink()
    # Convert back
    convert_trades_format(default_conf, convert_from='json',
                          convert_to='jsongz', erase=True)
+    for file in files:
+        assert file['old'].exists()
+        assert not file['new'].exists()

-    assert file.exists()
-    assert not file_new.exists()
-
-    _clean_test_file(file)
-    if file_new.exists():
-        file_new.unlink()
+        _clean_test_file(file['old'])
+        if file['new'].exists():
+            file['new'].unlink()


 def test_convert_ohlcv_format(mocker, default_conf, testdatadir):
--- a/tests/data/test_dataprovider.py
+++ b/tests/data/test_dataprovider.py
@ -1,8 +1,11 @@
 from unittest.mock import MagicMock

 from pandas import DataFrame
+import pytest

 from freqtrade.data.dataprovider import DataProvider
+from freqtrade.pairlist.pairlistmanager import PairListManager
+from freqtrade.exceptions import OperationalException
 from freqtrade.state import RunMode
 from tests.conftest import get_patched_exchange

@ -64,8 +67,8 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history):
    assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty

    # Test with and without parameter
-    assert dp.get_pair_dataframe("UNITTEST/BTC",
-                                 ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC"))
+    assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)\
+        .equals(dp.get_pair_dataframe("UNITTEST/BTC"))

    default_conf["runmode"] = RunMode.LIVE
    dp = DataProvider(default_conf, exchange)
@ -90,10 +93,7 @@ def test_available_pairs(mocker, default_conf, ohlcv_history):

    dp = DataProvider(default_conf, exchange)
    assert len(dp.available_pairs) == 2
-    assert dp.available_pairs == [
-        ("XRP/BTC", ticker_interval),
-        ("UNITTEST/BTC", ticker_interval),
-    ]
+    assert dp.available_pairs == [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval), ]


 def test_refresh(mocker, default_conf, ohlcv_history):
@ -152,3 +152,27 @@ def test_market(mocker, default_conf, markets):

    res = dp.market('UNITTEST/BTC')
    assert res is None
+
+
+def test_current_whitelist(mocker, default_conf, tickers):
+    # patch default conf to volumepairlist
+    default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5}
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers)
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    pairlist = PairListManager(exchange, default_conf)
+    dp = DataProvider(default_conf, exchange, pairlist)
+
+    # Simulate volumepairs from exchange.
+    pairlist.refresh_pairlist()
+
+    assert dp.current_whitelist() == pairlist._whitelist
+    # The identity of the 2 lists should be identical
+    assert dp.current_whitelist() is pairlist._whitelist
+
+    with pytest.raises(OperationalException):
+        dp = DataProvider(default_conf, exchange)
+        dp.current_whitelist()
--- a/tests/data/test_history.py
+++ b/tests/data/test_history.py
@ -547,6 +547,17 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
    assert log_has("New Amount of trades: 5", caplog)
    assert file1.is_file()

+    ght_mock.reset_mock()
+    since_time = int(trades_history[-3][0] // 1000)
+    since_time2 = int(trades_history[-1][0] // 1000)
+    timerange = TimeRange('date', None, since_time, 0)
+    assert _download_trades_history(data_handler=data_handler, exchange=exchange,
+                                    pair='ETH/BTC', timerange=timerange)
+
+    assert ght_mock.call_count == 1
+    # Check this in seconds - since we had to convert to seconds above too.
+    assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time2 - 5
+
    # clean files freshly downloaded
    _clean_test_file(file1)

@ -601,7 +612,7 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir):
 def test_jsondatahandler_trades_get_pairs(testdatadir):
    pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
    # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'XRP/ETH'}
+    assert set(pairs) == {'XRP/ETH', 'XRP/OLD'}


 def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
@ -614,6 +625,17 @@ def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m')


+def test_jsondatahandler_trades_load(mocker, testdatadir, caplog):
+    dh = JsonGzDataHandler(testdatadir)
+    logmsg = "Old trades format detected - converting"
+    dh.trades_load('XRP/ETH')
+    assert not log_has(logmsg, caplog)
+
+    # Test conversation is happening
+    dh.trades_load('XRP/OLD')
+    assert log_has(logmsg, caplog)
+
+
 def test_jsondatahandler_trades_purge(mocker, testdatadir):
    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
    mocker.patch.object(Path, "unlink", MagicMock())
--- a/tests/edge/test_edge.py
+++ b/tests/edge/test_edge.py
@ -335,12 +335,16 @@ def test_edge_init_error(mocker, edge_conf,):
        get_patched_freqtradebot(mocker, edge_conf)


-def test_process_expectancy(mocker, edge_conf):
+@pytest.mark.parametrize("fee,risk_reward_ratio,expectancy", [
+    (0.0005, 306.5384615384, 101.5128205128),
+    (0.001, 152.6923076923, 50.2307692308),
+])
+def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectancy):
    edge_conf['edge']['min_trade_number'] = 2
    freqtrade = get_patched_freqtradebot(mocker, edge_conf)

    def get_fee(*args, **kwargs):
-        return 0.001
+        return fee

    freqtrade.exchange.get_fee = get_fee
    edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
@ -394,9 +398,9 @@ def test_process_expectancy(mocker, edge_conf):
    assert 'TEST/BTC' in final
    assert final['TEST/BTC'].stoploss == -0.9
    assert round(final['TEST/BTC'].winrate, 10) == 0.3333333333
-    assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384
+    assert round(final['TEST/BTC'].risk_reward_ratio, 10) == risk_reward_ratio
    assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0
-    assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128
+    assert round(final['TEST/BTC'].expectancy, 10) == expectancy

    # Pop last item so no trade is profitable
    trades.pop()
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@ -1537,18 +1537,18 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na
@pytest.mark.asyncio
@pytest.mark.parametrize("exchange_name", EXCHANGES)
 async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name,
-                                   trades_history):
+                                   fetch_trades_result):

    caplog.set_level(logging.DEBUG)
    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
    # Monkey-patch async function
-    exchange._api_async.fetch_trades = get_mock_coro(trades_history)
+    exchange._api_async.fetch_trades = get_mock_coro(fetch_trades_result)

    pair = 'ETH/BTC'
    res = await exchange._async_fetch_trades(pair, since=None, params=None)
    assert type(res) is list
-    assert isinstance(res[0], dict)
-    assert isinstance(res[1], dict)
+    assert isinstance(res[0], list)
+    assert isinstance(res[1], list)

    assert exchange._api_async.fetch_trades.call_count == 1
    assert exchange._api_async.fetch_trades.call_args[0][0] == pair
@ -1594,7 +1594,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang
        if 'since' in kwargs:
            # Return first 3
            return trades_history[:-2]
-        elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3]['id']:
+        elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3][1]:
            # Return 2
            return trades_history[-3:-1]
        else:
@ -1604,8 +1604,8 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang
    exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)

    pair = 'ETH/BTC'
-    ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0]["timestamp"],
-                                                     until=trades_history[-1]["timestamp"]-1)
+    ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0][0],
+                                                     until=trades_history[-1][0]-1)
    assert type(ret) is tuple
    assert ret[0] == pair
    assert type(ret[1]) is list
@ -1614,7 +1614,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang
    fetch_trades_cal = exchange._async_fetch_trades.call_args_list
    # first call (using since, not fromId)
    assert fetch_trades_cal[0][0][0] == pair
-    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]

    # 2nd call
    assert fetch_trades_cal[1][0][0] == pair
@ -1630,7 +1630,7 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
    caplog.set_level(logging.DEBUG)

    async def mock_get_trade_hist(pair, *args, **kwargs):
-        if kwargs['since'] == trades_history[0]["timestamp"]:
+        if kwargs['since'] == trades_history[0][0]:
            return trades_history[:-1]
        else:
            return trades_history[-1:]
@ -1640,8 +1640,8 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
    # Monkey-patch async function
    exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
    pair = 'ETH/BTC'
-    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
-                                                       until=trades_history[-1]["timestamp"]-1)
+    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0],
+                                                       until=trades_history[-1][0]-1)
    assert type(ret) is tuple
    assert ret[0] == pair
    assert type(ret[1]) is list
@ -1650,11 +1650,11 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
    fetch_trades_cal = exchange._async_fetch_trades.call_args_list
    # first call (using since, not fromId)
    assert fetch_trades_cal[0][0][0] == pair
-    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]

    # 2nd call
    assert fetch_trades_cal[1][0][0] == pair
-    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]
    assert log_has_re(r"Stopping because until was reached.*", caplog)


@ -1666,7 +1666,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
    caplog.set_level(logging.DEBUG)

    async def mock_get_trade_hist(pair, *args, **kwargs):
-        if kwargs['since'] == trades_history[0]["timestamp"]:
+        if kwargs['since'] == trades_history[0][0]:
            return trades_history[:-1]
        else:
            return []
@ -1676,8 +1676,8 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
    # Monkey-patch async function
    exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
    pair = 'ETH/BTC'
-    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
-                                                       until=trades_history[-1]["timestamp"]-1)
+    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0],
+                                                       until=trades_history[-1][0]-1)
    assert type(ret) is tuple
    assert ret[0] == pair
    assert type(ret[1]) is list
@ -1686,7 +1686,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
    fetch_trades_cal = exchange._async_fetch_trades.call_args_list
    # first call (using since, not fromId)
    assert fetch_trades_cal[0][0][0] == pair
-    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]


@pytest.mark.parametrize("exchange_name", EXCHANGES)
@ -1698,8 +1698,8 @@ def test_get_historic_trades(default_conf, mocker, caplog, exchange_name, trades

    exchange._async_get_trade_history_id = get_mock_coro((pair, trades_history))
    exchange._async_get_trade_history_time = get_mock_coro((pair, trades_history))
-    ret = exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
-                                       until=trades_history[-1]["timestamp"])
+    ret = exchange.get_historic_trades(pair, since=trades_history[0][0],
+                                       until=trades_history[-1][0])

    # Depending on the exchange, one or the other method should be called
    assert sum([exchange._async_get_trade_history_id.call_count,
@ -1720,8 +1720,8 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange

    with pytest.raises(OperationalException,
                       match="This exchange does not suport downloading Trades."):
-        exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
-                                     until=trades_history[-1]["timestamp"])
+        exchange.get_historic_trades(pair, since=trades_history[0][0],
+                                     until=trades_history[-1][0])


@pytest.mark.parametrize("exchange_name", EXCHANGES)
--- a/tests/optimize/test_backtesting.py
+++ b/tests/optimize/test_backtesting.py
@ -649,6 +649,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
        assert log_has(line, caplog)


+@pytest.mark.filterwarnings("ignore:deprecated")
 def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):

    patch_exchange(mocker)
--- a/tests/rpc/test_rpc_apiserver.py
+++ b/tests/rpc/test_rpc_apiserver.py
@ -94,6 +94,33 @@ def test_api_unauthorized(botclient):
    assert rc.json == {'error': 'Unauthorized'}


+def test_api_token_login(botclient):
+    ftbot, client = botclient
+    rc = client_post(client, f"{BASE_URI}/token/login")
+    assert_response(rc)
+    assert 'access_token' in rc.json
+    assert 'refresh_token' in rc.json
+
+    # test Authentication is working with JWT tokens too
+    rc = client.get(f"{BASE_URI}/count",
+                    content_type="application/json",
+                    headers={'Authorization': f'Bearer {rc.json["access_token"]}'})
+    assert_response(rc)
+
+
+def test_api_token_refresh(botclient):
+    ftbot, client = botclient
+    rc = client_post(client, f"{BASE_URI}/token/login")
+    assert_response(rc)
+    rc = client.post(f"{BASE_URI}/token/refresh",
+                     content_type="application/json",
+                     data=None,
+                     headers={'Authorization': f'Bearer {rc.json["refresh_token"]}'})
+    assert_response(rc)
+    assert 'access_token' in rc.json
+    assert 'refresh_token' not in rc.json
+
+
 def test_api_stop_workflow(botclient):
    ftbot, client = botclient
    assert ftbot.state == State.RUNNING
@ -123,6 +150,12 @@ def test_api__init__(default_conf, mocker):
    """
    Test __init__() method
    """
+    default_conf.update({"api_server": {"enabled": True,
+                                        "listen_ip_address": "127.0.0.1",
+                                        "listen_port": 8080,
+                                        "username": "TestUser",
+                                        "password": "testPass",
+                                        }})
    mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock())
    mocker.patch('freqtrade.rpc.api_server.ApiServer.run', MagicMock())

@ -283,6 +316,7 @@ def test_api_show_config(botclient, mocker):
    assert 'dry_run' in rc.json
    assert rc.json['exchange'] == 'bittrex'
    assert rc.json['ticker_interval'] == '5m'
+    assert rc.json['state'] == 'running'
    assert not rc.json['trailing_stop']


--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@ -2313,19 +2313,41 @@ def test_handle_timedout_limit_buy(mocker, caplog, default_conf, limit_buy_order
    Trade.session = MagicMock()
    trade = MagicMock()
    trade.pair = 'LTC/ETH'
-    limit_buy_order['remaining'] = limit_buy_order['amount']
+    limit_buy_order['filled'] = 0.0
+    limit_buy_order['status'] = 'open'
    assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
    assert cancel_order_mock.call_count == 1

    cancel_order_mock.reset_mock()
-    limit_buy_order['amount'] = 2
+    limit_buy_order['filled'] = 2
    assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
    assert cancel_order_mock.call_count == 1

+    limit_buy_order['filled'] = 2
    mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException)
    assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)


+@pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'],
+                         indirect=['limit_buy_order_canceled_empty'])
+def test_handle_timedout_limit_buy_exchanges(mocker, caplog, default_conf,
+                                             limit_buy_order_canceled_empty) -> None:
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    cancel_order_mock = mocker.patch(
+        'freqtrade.exchange.Exchange.cancel_order_with_result',
+        return_value=limit_buy_order_canceled_empty)
+
+    freqtrade = FreqtradeBot(default_conf)
+
+    Trade.session = MagicMock()
+    trade = MagicMock()
+    trade.pair = 'LTC/ETH'
+    assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order_canceled_empty)
+    assert cancel_order_mock.call_count == 0
+    assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog)
+
+
@pytest.mark.parametrize('cancelorder', [
    {},
    {'remaining': None},
@ -2347,12 +2369,14 @@ def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_
    Trade.session = MagicMock()
    trade = MagicMock()
    trade.pair = 'LTC/ETH'
-    limit_buy_order['remaining'] = limit_buy_order['amount']
+    limit_buy_order['filled'] = 0.0
+    limit_buy_order['status'] = 'open'
+
    assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
    assert cancel_order_mock.call_count == 1

    cancel_order_mock.reset_mock()
-    limit_buy_order['amount'] = 2
+    limit_buy_order['filled'] = 1.0
    assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
    assert cancel_order_mock.call_count == 1

@ -2381,6 +2405,21 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None:
    assert cancel_order_mock.call_count == 1


+def test_handle_timedout_limit_sell_cancel_exception(mocker, default_conf) -> None:
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException())
+
+    freqtrade = FreqtradeBot(default_conf)
+
+    trade = MagicMock()
+    order = {'remaining': 1,
+             'amount': 1,
+             'status': "open"}
+    assert freqtrade.handle_timedout_limit_sell(trade, order) == 'error cancelling order'
+
+
 def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None:
    rpc_mock = patch_RPCManager(mocker)
    patch_exchange(mocker)
--- a/tests/test_main.py
+++ b/tests/test_main.py
@ -115,6 +115,32 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None:
    assert log_has('Oh snap!', caplog)


+def test_main_operational_exception1(mocker, default_conf, caplog) -> None:
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.commands.list_commands.available_exchanges',
+        MagicMock(side_effect=ValueError('Oh snap!'))
+    )
+    patched_configuration_load_config_file(mocker, default_conf)
+
+    args = ['list-exchanges']
+
+    # Test Main + the KeyboardInterrupt exception
+    with pytest.raises(SystemExit):
+        main(args)
+
+    assert log_has('Fatal exception!', caplog)
+    assert not log_has_re(r'SIGINT.*', caplog)
+    mocker.patch(
+        'freqtrade.commands.list_commands.available_exchanges',
+        MagicMock(side_effect=KeyboardInterrupt)
+    )
+    with pytest.raises(SystemExit):
+        main(args)
+
+    assert log_has_re(r'SIGINT.*', caplog)
+
+
 def test_main_reload_conf(mocker, default_conf, caplog) -> None:
    patch_exchange(mocker)
    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cleanup', MagicMock())
--- a/tests/testdata/XRP_ETH-trades.json.gz
+++ b/tests/testdata/XRP_ETH-trades.json.gz
--- a/tests/testdata/XRP_OLD-trades.json.gz
+++ b/tests/testdata/XRP_OLD-trades.json.gz