Merge branch 'develop' into pr/jpribyl/3210-1

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.2-slim-buster
+FROM python:3.8.3-slim-buster
 
 RUN apt-get update \
     && apt-get -y install curl build-essential libssl-dev \

build_helpers/install_windows.ps1

@@ -7,10 +7,10 @@ python -m pip install --upgrade pip
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
 
 if ($pyv -eq '3.7') {
-    pip install build_helpers\TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl
 }
 if ($pyv -eq '3.8') {
-    pip install build_helpers\TA_Lib-0.4.17-cp38-cp38-win_amd64.whl
+    pip install build_helpers\TA_Lib-0.4.18-cp38-cp38-win_amd64.whl
 }
 
 pip install -r requirements-dev.txt

config_full.json.example

@@ -121,6 +121,7 @@
         "enabled": false,
         "listen_ip_address": "127.0.0.1",
         "listen_port": 8080,
+        "jwt_secret_key": "somethingrandom",
         "username": "freqtrader",
         "password": "SuperSecurePassword"
     },

docs/configuration.md

@@ -109,7 +109,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below. <br> **Datatype:** Boolean
 | `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`. <br> **Datatype:** ClassName
 | `strategy_path` | Adds an additional strategy lookup path (must be a directory). <br> **Datatype:** String
-| `internals.process_throttle_secs` | Set the process throttle. Value in second. <br>*Defaults to `5` seconds.* <br> **Datatype:** Positive Intege
+| `internals.process_throttle_secs` | Set the process throttle. Value in second. <br>*Defaults to `5` seconds.* <br> **Datatype:** Positive Integer
 | `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages. <br>*Defaults to `60` seconds.* <br> **Datatype:** Positive Integer or 0
 | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. <br> **Datatype:** Boolean
 | `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file. <br> **Datatype:** String

docs/docker.md

@@ -22,6 +22,9 @@ Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.co
 !!! Note
     All below comands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file.
 
+!!! Note "Docker on Raspberry"
+    If you're running freqtrade on a Raspberry PI, you must change the image from `freqtradeorg/freqtrade:master` to `freqtradeorg/freqtrade:master_pi` or `freqtradeorg/freqtrade:develop_pi`, otherwise the image will not work.
+
 ### Docker quick start
 
 Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory.
@@ -65,7 +68,7 @@ docker-compose up -d
 
 #### Docker-compose logs
 
-Logs will be written to `user_data/freqtrade.log`. 
+Logs will be written to `user_data/logs/freqtrade.log`. 
 Alternatively, you can check the latest logs using `docker-compose logs -f`.
 
 #### Database

docs/installation.md

@@ -248,14 +248,14 @@ git clone https://github.com/freqtrade/freqtrade.git
 
 Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
 
-As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl` (make sure to use the version matching your python version)
+As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version)
 
 ```cmd
 >cd \path\freqtrade-develop
 >python -m venv .env
 >.env\Scripts\activate.bat
 REM optionally install ta-lib from wheel
-REM >pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl
+REM >pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl
 >pip install -r requirements.txt
 >pip install -e .
 >freqtrade

docs/requirements-docs.txt

@@ -1,2 +1,2 @@
-mkdocs-material==5.1.3
+mkdocs-material==5.1.6
 mdx_truly_sane_lists==1.2

docs/rest-api.md

@@ -11,6 +11,7 @@ Sample configuration:
         "enabled": true,
         "listen_ip_address": "127.0.0.1",
         "listen_port": 8080,
+        "jwt_secret_key": "somethingrandom",
         "username": "Freqtrader",
         "password": "SuperSecret1!"
     },
@@ -29,7 +30,7 @@ This should return the response:
 {"status":"pong"}
 ```
 
-All other endpoints return sensitive info and require authentication, so are not available through a web browser.
+All other endpoints return sensitive info and require authentication and are therefore not available through a web browser.
 
 To generate a secure password, either use a password manager, or use the below code snipped.
 
@@ -38,6 +39,9 @@ import secrets
 secrets.token_hex()
 ```
 
+!!! Hint
+    Use the same method to also generate a JWT secret key (`jwt_secret_key`).
+
 ### Configuration with docker
 
 If you run your bot using docker, you'll need to have the bot listen to incomming connections. The security is then handled by docker.
@@ -202,3 +206,28 @@ whitelist
         Show the current whitelist
         :returns: json object
 ```
+
+## Advanced API usage using JWT tokens
+
+!!! Note
+    The below should be done in an application (a Freqtrade REST API client, which fetches info via API), and is not intended to be used on a regular basis.
+
+Freqtrade's REST API also offers JWT (JSON Web Tokens).
+You can login using the following command, and subsequently use the resulting access_token.
+
+``` bash
+> curl -X POST --user Freqtrader http://localhost:8080/api/v1/token/login
+{"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiMmEwYmY0NWUtMjhmOS00YTUzLTlmNzItMmM5ZWVlYThkNzc2IiwiZXhwIjoxNTg5MTIwNTgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.qt6MAXYIa-l556OM7arBvYJ0SDI9J8bIk3_glDujF5g","refresh_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiZWQ1ZWI3YjAtYjMwMy00YzAyLTg2N2MtNWViMjIxNWQ2YTMxIiwiZXhwIjoxNTkxNzExNjgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJ0eXBlIjoicmVmcmVzaCJ9.d1AT_jYICyTAjD0fiQAr52rkRqtxCjUGEMwlNuuzgNQ"}
+
+> access_token="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiMmEwYmY0NWUtMjhmOS00YTUzLTlmNzItMmM5ZWVlYThkNzc2IiwiZXhwIjoxNTg5MTIwNTgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.qt6MAXYIa-l556OM7arBvYJ0SDI9J8bIk3_glDujF5g"
+# Use access_token for authentication
+> curl -X GET --header "Authorization: Bearer ${access_token}" http://localhost:8080/api/v1/count
+
+```
+
+Since the access token has a short timeout (15 min) - the `token/refresh` request should be used periodically to get a fresh access token:
+
+``` bash
+> curl -X POST --header "Authorization: Bearer ${refresh_token}"http://localhost:8080/api/v1/token/refresh
+{"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk5NzQsIm5iZiI6MTU4OTExOTk3NCwianRpIjoiMDBjNTlhMWUtMjBmYS00ZTk0LTliZjAtNWQwNTg2MTdiZDIyIiwiZXhwIjoxNTg5MTIwODc0LCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.1seHlII3WprjjclY6DpRhen0rqdF4j6jbvxIhUFaSbs"}
+```

docs/sql_cheatsheet.md

@@ -1,13 +1,20 @@
 # SQL Helper
+
 This page contains some help if you want to edit your sqlite db.
 
 ## Install sqlite3
-**Ubuntu/Debian installation**
+
+Sqlite3 is a terminal based sqlite application.
+Feel free to use a visual Database editor like SqliteBrowser if you feel more comfortable with that.
+
+### Ubuntu/Debian installation
+
 ```bash
 sudo apt-get install sqlite3
 ```
 
 ## Open the DB
+
 ```bash
 sqlite3
 .open <filepath>
@@ -16,45 +23,61 @@ sqlite3
 ## Table structure
 
 ### List tables
+
 ```bash
 .tables
 ```
 
 ### Display table structure
+
 ```bash
 .schema <table_name>
 ```
 
 ### Trade table structure
+
 ```sql
-CREATE TABLE trades (
+CREATE TABLE trades
     id INTEGER NOT NULL,
     exchange VARCHAR NOT NULL,
     pair VARCHAR NOT NULL,
     is_open BOOLEAN NOT NULL,
     fee_open FLOAT NOT NULL,
+    fee_open_cost FLOAT,
+    fee_open_currency VARCHAR,
     fee_close FLOAT NOT NULL,
+    fee_close_cost FLOAT,
+    fee_close_currency VARCHAR,
     open_rate FLOAT,
     open_rate_requested FLOAT,
+    open_trade_price FLOAT,
     close_rate FLOAT,
     close_rate_requested FLOAT,
     close_profit FLOAT,
+    close_profit_abs FLOAT,
     stake_amount FLOAT NOT NULL,
     amount FLOAT,
     open_date DATETIME NOT NULL,
     close_date DATETIME,
     open_order_id VARCHAR,
     stop_loss FLOAT,
+    stop_loss_pct FLOAT,
     initial_stop_loss FLOAT,
+    initial_stop_loss_pct FLOAT,
     stoploss_order_id VARCHAR,
     stoploss_last_update DATETIME,
     max_rate FLOAT,
+    min_rate FLOAT,
     sell_reason VARCHAR,
     strategy VARCHAR,
     ticker_interval INTEGER,
     PRIMARY KEY (id),
     CHECK (is_open IN (0, 1))
 );
+CREATE INDEX ix_trades_stoploss_order_id ON trades (stoploss_order_id);
+CREATE INDEX ix_trades_pair ON trades (pair);
+CREATE INDEX ix_trades_is_open ON trades (is_open);
+
 ```
 
 ## Get all trades in the table

docs/strategy-advanced.md

@@ -20,7 +20,7 @@ It applies a tight timeout for higher priced assets, while allowing more time to
 The function must return either `True` (cancel order) or `False` (keep order alive).
 
 ``` python
-from datetime import datetime, timestamp
+from datetime import datetime, timedelta
 from freqtrade.persistence import Trade
 
 class Awesomestrategy(IStrategy):
@@ -59,7 +59,7 @@ class Awesomestrategy(IStrategy):
 ### Custom order timeout example (using additional data)
 
 ``` python
-from datetime import datetime, timestamp
+from datetime import datetime
 from freqtrade.persistence import Trade
 
 class Awesomestrategy(IStrategy):

docs/strategy-customization.md

@@ -324,67 +324,14 @@ class Awesomestrategy(IStrategy):
 !!! Note
     If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
-### Additional data (DataProvider)
+***
 
-The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy.
-
-All methods return `None` in case of failure (do not raise an exception).
-
-Please always check the mode of operation to select the correct method to get data (samples see below).
-
-#### Possible options for DataProvider
-
-- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval).
-- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame.
-- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk.
-- `get_pair_dataframe(pair, timeframe)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes).
-- `orderbook(pair, maximum)` - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries.
-- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure.
-- `runmode` - Property containing the current runmode.
-
-#### Example: fetch live / historical candle (OHLCV) data for the first informative pair
-
-``` python
-if self.dp:
-    inf_pair, inf_timeframe = self.informative_pairs()[0]
-    informative = self.dp.get_pair_dataframe(pair=inf_pair,
-                                             timeframe=inf_timeframe)
-```
-
-!!! Warning "Warning about backtesting"
-    Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
-    for the backtesting runmode) provides the full time-range in one go,
-    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode).
-
-!!! Warning "Warning in hyperopt"
-    This option cannot currently be used during hyperopt.
-
-#### Orderbook
-
-``` python
-if self.dp:
-    if self.dp.runmode.value in ('live', 'dry_run'):
-        ob = self.dp.orderbook(metadata['pair'], 1)
-        dataframe['best_bid'] = ob['bids'][0][0]
-        dataframe['best_ask'] = ob['asks'][0][0]
-```
-
-!!! Warning
-    The order book is not part of the historic data which means backtesting and hyperopt will not work if this
-    method is used.
-
-#### Available Pairs
-
-``` python
-if self.dp:
-    for pair, timeframe in self.dp.available_pairs:
-        print(f"available {pair}, {timeframe}")
-```
+### Additional data (informative_pairs)
 
 #### Get data for non-tradeable pairs
 
 Data for additional, informative pairs (reference pairs) can be beneficial for some strategies.
-Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see above).
+Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below).
 These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting.
 
 The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument.
@@ -404,6 +351,125 @@ def informative_pairs(self):
     It is however better to use resampling to longer time-intervals when possible
     to avoid hammering the exchange with too many requests and risk being blocked.
 
+***
+
+### Additional data (DataProvider)
+
+The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy.
+
+All methods return `None` in case of failure (do not raise an exception).
+
+Please always check the mode of operation to select the correct method to get data (samples see below).
+
+#### Possible options for DataProvider
+
+- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval).
+- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist)
+- [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes).
+- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk.
+- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on the Market data structure.
+- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame.
+- [`orderbook(pair, maximum)`](#orderbookpair-maximum) - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries.
+- [`ticker(pair)`](#tickerpair) - Returns current ticker data for the pair. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#price-tickers) for more details on the Ticker data structure.
+- `runmode` - Property containing the current runmode.
+
+#### Example Usages:
+
+#### *available_pairs*
+
+``` python
+if self.dp:
+    for pair, timeframe in self.dp.available_pairs:
+        print(f"available {pair}, {timeframe}")
+```
+
+#### *current_whitelist()*
+Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. 
+
+The strategy might look something like this:
+
+*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day ATR to buy and sell.*
+
+Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least!
+
+Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use.
+
+This is where calling `self.dp.current_whitelist()` comes in handy.
+
+```python
+class SampleStrategy(IStrategy):
+    # strategy init stuff...
+
+    ticker_interval = '5m'
+
+    # more strategy init stuff..
+
+    def informative_pairs(self):
+
+        # get access to all pairs available in whitelist. 
+        pairs = self.dp.current_whitelist()
+        # Assign tf to each pair so they can be downloaded and cached for strategy.
+        informative_pairs = [(pair, '1d') for pair in pairs]
+        return informative_pairs
+
+    def populate_indicators(self, dataframe, metadata):
+        # Get the informative pair
+        informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d')
+        # Get the 14 day ATR.
+        atr = ta.ATR(informative, timeperiod=14)
+        # Do other stuff
+```
+
+#### *get_pair_dataframe(pair, timeframe)*
+
+``` python
+# fetch live / historical candle (OHLCV) data for the first informative pair
+if self.dp:
+    inf_pair, inf_timeframe = self.informative_pairs()[0]
+    informative = self.dp.get_pair_dataframe(pair=inf_pair,
+                                             timeframe=inf_timeframe)
+```
+
+!!! Warning "Warning about backtesting"
+    Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()`
+    for the backtesting runmode) provides the full time-range in one go,
+    so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode).
+
+!!! Warning "Warning in hyperopt"
+    This option cannot currently be used during hyperopt.
+
+#### *orderbook(pair, maximum)*
+
+``` python
+if self.dp:
+    if self.dp.runmode.value in ('live', 'dry_run'):
+        ob = self.dp.orderbook(metadata['pair'], 1)
+        dataframe['best_bid'] = ob['bids'][0][0]
+        dataframe['best_ask'] = ob['asks'][0][0]
+```
+
+!!! Warning
+    The order book is not part of the historic data which means backtesting and hyperopt will not work if this
+    method is used.
+
+#### *ticker(pair)*
+
+``` python
+if self.dp:
+    if self.dp.runmode.value in ('live', 'dry_run'):
+        ticker = self.dp.ticker(metadata['pair'])
+        dataframe['last_price'] = ticker['last']
+        dataframe['volume24h'] = ticker['quoteVolume']
+        dataframe['vwap'] = ticker['vwap']
+```
+
+!!! Warning
+    Although the ticker data structure is a part of the ccxt Unified Interface, the values returned by this method can
+    vary for different exchanges. For instance, many exchanges do not return `vwap` values, the FTX exchange
+    does not always fills in the `last` field (so it can be None), etc. So you need to carefully verify the ticker
+    data returned from the exchange and add appropriate error handling / defaults.
+
+***
 ### Additional data (Wallets)
 
 The strategy provides access to the `Wallets` object. This contains the current balances on the exchange.
@@ -426,6 +492,7 @@ if self.wallets:
 - `get_used(asset)` - currently tied up balance (open orders)
 - `get_total(asset)` - total available balance - sum of the 2 above
 
+***
 ### Additional data (Trades)
 
 A history of Trades can be retrieved in the strategy by querying the database.

docs/utils.md

@@ -521,3 +521,48 @@ Prints JSON data with details for the last best epoch (i.e., the best of all epo
 ```
 freqtrade hyperopt-show --best -n -1 --print-json --no-header
 ```
+
+## Show trades
+
+Print selected (or all) trades from database to screen.
+
+```
+usage: freqtrade show-trades [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+                             [-d PATH] [--userdir PATH] [--db-url PATH]
+                             [--trade-ids TRADE_IDS [TRADE_IDS ...]]
+                             [--print-json]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --db-url PATH         Override trades database URL, this is useful in custom
+                        deployments (default: `sqlite:///tradesv3.sqlite` for
+                        Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for
+                        Dry Run).
+  --trade-ids TRADE_IDS [TRADE_IDS ...]
+                        Specify the list of trade ids.
+  --print-json          Print output in JSON format.
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+```
+
+### Examples
+
+Print trades with id 2 and 3 as json
+
+``` bash
+freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print-json
+```

freqtrade/commands/__init__.py

@@ -19,7 +19,8 @@ from freqtrade.commands.list_commands import (start_list_exchanges,
                                               start_list_hyperopts,
                                               start_list_markets,
                                               start_list_strategies,
-                                              start_list_timeframes)
+                                              start_list_timeframes,
+                                              start_show_trades)
 from freqtrade.commands.optimize_commands import (start_backtesting,
                                                   start_edge, start_hyperopt)
 from freqtrade.commands.pairlist_commands import start_test_pairlist

freqtrade/commands/arguments.py

@@ -64,6 +64,8 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
                     "trade_source", "ticker_interval"]
 
+ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
+
 ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
                       "hyperopt_list_min_trades", "hyperopt_list_max_trades",
                       "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time",
@@ -78,7 +80,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
 NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
                     "list-markets", "list-pairs", "list-strategies",
                     "list-hyperopts", "hyperopt-list", "hyperopt-show",
-                    "plot-dataframe", "plot-profit"]
+                    "plot-dataframe", "plot-profit", "show-trades"]
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"]
 
@@ -163,7 +165,7 @@ class Arguments:
                                         start_list_markets, start_list_strategies,
                                         start_list_timeframes, start_new_config,
                                         start_new_hyperopt, start_new_strategy,
-                                        start_plot_dataframe, start_plot_profit,
+                                        start_plot_dataframe, start_plot_profit, start_show_trades,
                                         start_backtesting, start_hyperopt, start_edge,
                                         start_test_pairlist, start_trading)
 
@@ -330,6 +332,15 @@ class Arguments:
         plot_profit_cmd.set_defaults(func=start_plot_profit)
         self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
 
+        # Add show-trades subcommand
+        show_trades = subparsers.add_parser(
+            'show-trades',
+            help='Show trades.',
+            parents=[_common_parser],
+        )
+        show_trades.set_defaults(func=start_show_trades)
+        self._build_args(optionlist=ARGS_SHOW_TRADES, parser=show_trades)
+
         # Add hyperopt-list subcommand
         hyperopt_list_cmd = subparsers.add_parser(
             'hyperopt-list',

freqtrade/commands/cli_options.py

@@ -222,7 +222,7 @@ AVAILABLE_CLI_OPTIONS = {
     ),
     "print_json": Arg(
         '--print-json',
-        help='Print best result detailization in JSON format.',
+        help='Print output in JSON format.',
         action='store_true',
         default=False,
     ),
@@ -430,6 +430,11 @@ AVAILABLE_CLI_OPTIONS = {
         choices=["DB", "file"],
         default="file",
     ),
+    "trade_ids": Arg(
+        '--trade-ids',
+        help='Specify the list of trade ids.',
+        nargs='+',
+    ),
     # hyperopt-list, hyperopt-show
     "hyperopt_list_profitable": Arg(
         '--profitable',

freqtrade/commands/list_commands.py

@@ -197,3 +197,30 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
                   args.get('list_pairs_print_json', False) or
                   args.get('print_csv', False)):
             print(f"{summary_str}.")
+
+
+def start_show_trades(args: Dict[str, Any]) -> None:
+    """
+    Show trades
+    """
+    from freqtrade.persistence import init, Trade
+    import json
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    if 'db_url' not in config:
+        raise OperationalException("--db-url is required for this command.")
+
+    logger.info(f'Using DB: "{config["db_url"]}"')
+    init(config['db_url'], clean_open_orders=False)
+    tfilter = []
+
+    if config.get('trade_ids'):
+        tfilter.append(Trade.id.in_(config['trade_ids']))
+
+    trades = Trade.get_trades(tfilter).all()
+    logger.info(f"Printing {len(trades)} Trades: ")
+    if config.get('print_json', False):
+        print(json.dumps([trade.to_json() for trade in trades], indent=4))
+    else:
+        for trade in trades:
+            print(trade)

freqtrade/commands/trade_commands.py

@@ -18,6 +18,9 @@ def start_trading(args: Dict[str, Any]) -> int:
     try:
         worker = Worker(args)
         worker.run()
+    except Exception as e:
+        logger.error(str(e))
+        logger.exception("Fatal exception!")
     except KeyboardInterrupt:
         logger.info('SIGINT received, aborting ...')
     finally:

freqtrade/configuration/configuration.py

@@ -356,8 +356,12 @@ class Configuration:
         self._args_to_config(config, argname='indicators2',
                              logstring='Using indicators2: {}')
 
+        self._args_to_config(config, argname='trade_ids',
+                             logstring='Filtering on trade_ids: {}')
+
         self._args_to_config(config, argname='plot_limit',
                              logstring='Limiting plot to: {}')
+
         self._args_to_config(config, argname='trade_source',
                              logstring='Using trades from: {}')
 

freqtrade/constants.py

@@ -24,6 +24,9 @@ AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
 DRY_RUN_WALLET = 1000
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons
 DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
+# Don't modify sequence of DEFAULT_TRADES_COLUMNS
+# it has wide consequences for stored trades files
+DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
 
 USERPATH_HYPEROPTS = 'hyperopts'
 USERPATH_STRATEGIES = 'strategies'

freqtrade/data/converter.py

@@ -1,14 +1,17 @@
 """
 Functions to convert data from one format to another
 """
+import itertools
 import logging
 from datetime import datetime, timezone
-from typing import Any, Dict
+from operator import itemgetter
+from typing import Any, Dict, List
 
 import pandas as pd
 from pandas import DataFrame, to_datetime
 
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
+from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
+                                 DEFAULT_TRADES_COLUMNS)
 
 logger = logging.getLogger(__name__)
 
@@ -154,7 +157,27 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
     return frame
 
 
-def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
+def trades_remove_duplicates(trades: List[List]) -> List[List]:
+    """
+    Removes duplicates from the trades list.
+    Uses itertools.groupby to avoid converting to pandas.
+    Tests show it as being pretty efficient on lists of 4M Lists.
+    :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns
+    :return: same format as above, but with duplicates removed
+    """
+    return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))]
+
+
+def trades_dict_to_list(trades: List[Dict]) -> List[List]:
+    """
+    Convert fetch_trades result into a List (to be more memory efficient).
+    :param trades: List of trades, as returned by ccxt.fetch_trades.
+    :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns
+    """
+    return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades]
+
+
+def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame:
     """
     Converts trades list to OHLCV list
     TODO: This should get a dedicated test
@@ -164,9 +187,10 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
     """
     from freqtrade.exchange import timeframe_to_minutes
     timeframe_minutes = timeframe_to_minutes(timeframe)
-    df = pd.DataFrame(trades)
-    df['datetime'] = pd.to_datetime(df['datetime'])
-    df = df.set_index('datetime')
+    df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS)
+    df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms',
+                                     utc=True,)
+    df = df.set_index('timestamp')
 
     df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
     df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()

freqtrade/data/dataprovider.py

@@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple
 from pandas import DataFrame
 
 from freqtrade.data.history import load_pair_history
+from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.state import RunMode
 
@@ -18,9 +19,10 @@ logger = logging.getLogger(__name__)
 
 class DataProvider:
 
-    def __init__(self, config: dict, exchange: Exchange) -> None:
+    def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None:
         self._config = config
         self._exchange = exchange
+        self._pairlists = pairlists
 
     def refresh(self,
                 pairlist: List[Tuple[str, str]],
@@ -95,10 +97,14 @@ class DataProvider:
 
     def ticker(self, pair: str):
         """
-        Return last ticker data
+        Return last ticker data from exchange
+        :param pair: Pair to get the data for
+        :return: Ticker dict from exchange or empty dict if ticker is not available for the pair
         """
-        # TODO: Implement me
-        pass
+        try:
+            return self._exchange.fetch_ticker(pair)
+        except DependencyException:
+            return {}
 
     def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
         """
@@ -116,3 +122,17 @@ class DataProvider:
         can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other".
         """
         return RunMode(self._config.get('runmode', RunMode.OTHER))
+
+    def current_whitelist(self) -> List[str]:
+        """
+        fetch latest available whitelist.
+
+        Useful when you have a large whitelist and need to call each pair as an informative pair.
+        As available pairs does not show whitelist until after informative pairs have been cached.
+        :return: list of pairs in whitelist
+        """
+
+        if self._pairlists:
+            return self._pairlists.whitelist
+        else:
+            raise OperationalException("Dataprovider was not initialized with a pairlist provider.")

freqtrade/data/history/history_utils.py

@@ -9,10 +9,13 @@ from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
-from freqtrade.data.converter import ohlcv_to_dataframe, trades_to_ohlcv
+from freqtrade.data.converter import (ohlcv_to_dataframe,
+                                      trades_remove_duplicates,
+                                      trades_to_ohlcv)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
+from freqtrade.misc import format_ms_time
 
 logger = logging.getLogger(__name__)
 
@@ -257,27 +260,40 @@ def _download_trades_history(exchange: Exchange,
     """
     try:
 
-        since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None
+        since = timerange.startts * 1000 if \
+            (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
+                days=-30).float_timestamp) * 1000
 
         trades = data_handler.trades_load(pair)
 
-        from_id = trades[-1]['id'] if trades else None
+        # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
+        # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+        # DEFAULT_TRADES_COLUMNS: 1 -> id
 
-        logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
-        logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
+        from_id = trades[-1][1] if trades else None
+        if trades and since < trades[-1][0]:
+            # Reset since to the last available point
+            # - 5 seconds (to ensure we're getting all trades)
+            since = trades[-1][0] - (5 * 1000)
+            logger.info(f"Using last trade date -5s - Downloading trades for {pair} "
+                        f"since: {format_ms_time(since)}.")
+
+        logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}")
+        logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}")
+        logger.info(f"Current Amount of trades: {len(trades)}")
 
         # Default since_ms to 30 days if nothing is given
         new_trades = exchange.get_historic_trades(pair=pair,
-                                                  since=since if since else
-                                                  int(arrow.utcnow().shift(
-                                                      days=-30).float_timestamp) * 1000,
+                                                  since=since,
                                                   from_id=from_id,
                                                   )
         trades.extend(new_trades[1])
+        # Remove duplicates to make sure we're not storing data we don't need
+        trades = trades_remove_duplicates(trades)
         data_handler.trades_store(pair, data=trades)
 
-        logger.debug("New Start: %s", trades[0]['datetime'])
-        logger.debug("New End: %s", trades[-1]['datetime'])
+        logger.debug(f"New Start: {format_ms_time(trades[0][0])}")
+        logger.debug(f"New End: {format_ms_time(trades[-1][0])}")
         logger.info(f"New Amount of trades: {len(trades)}")
         return True
 

freqtrade/data/history/idatahandler.py

@@ -8,16 +8,20 @@ from abc import ABC, abstractclassmethod, abstractmethod
 from copy import deepcopy
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Dict, List, Optional, Type
+from typing import List, Optional, Type
 
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
-from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
+from freqtrade.data.converter import (clean_ohlcv_dataframe,
+                                      trades_remove_duplicates, trim_dataframe)
 from freqtrade.exchange import timeframe_to_seconds
 
 logger = logging.getLogger(__name__)
 
+# Type for trades list
+TradeList = List[List]
+
 
 class IDataHandler(ABC):
 
@@ -89,23 +93,25 @@ class IDataHandler(ABC):
         """
 
     @abstractmethod
-    def trades_store(self, pair: str, data: List[Dict]) -> None:
+    def trades_store(self, pair: str, data: TradeList) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
-        :param data: List of Dicts containing trade data
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
         """
 
     @abstractmethod
-    def trades_append(self, pair: str, data: List[Dict]):
+    def trades_append(self, pair: str, data: TradeList):
         """
         Append data to existing files
         :param pair: Pair - used for filename
-        :param data: List of Dicts containing trade data
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
         """
 
     @abstractmethod
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
         """
         Load a pair from file, either .json.gz or .json
         :param pair: Load trades for this pair
@@ -121,6 +127,16 @@ class IDataHandler(ABC):
         :return: True when deleted, false if file did not exist.
         """
 
+    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+        """
+        Load a pair from file, either .json.gz or .json
+        Removes duplicates in the process.
+        :param pair: Load trades for this pair
+        :param timerange: Timerange to load trades for - currently not implemented
+        :return: List of trades
+        """
+        return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
+
     def ohlcv_load(self, pair, timeframe: str,
                    timerange: Optional[TimeRange] = None,
                    fill_missing: bool = True,

freqtrade/data/history/jsondatahandler.py

@@ -1,6 +1,7 @@
+import logging
 import re
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 import numpy as np
 from pandas import DataFrame, read_json, to_datetime
@@ -8,8 +9,11 @@ from pandas import DataFrame, read_json, to_datetime
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
+from freqtrade.data.converter import trades_dict_to_list
 
-from .idatahandler import IDataHandler
+from .idatahandler import IDataHandler, TradeList
+
+logger = logging.getLogger(__name__)
 
 
 class JsonDataHandler(IDataHandler):
@@ -113,24 +117,26 @@ class JsonDataHandler(IDataHandler):
         # Check if regex found something and only return these results to avoid exceptions.
         return [match[0].replace('_', '/') for match in _tmp if match]
 
-    def trades_store(self, pair: str, data: List[Dict]) -> None:
+    def trades_store(self, pair: str, data: TradeList) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
-        :param data: List of Dicts containing trade data
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
         """
         filename = self._pair_trades_filename(self._datadir, pair)
         misc.file_dump_json(filename, data, is_zip=self._use_zip)
 
-    def trades_append(self, pair: str, data: List[Dict]):
+    def trades_append(self, pair: str, data: TradeList):
         """
         Append data to existing files
         :param pair: Pair - used for filename
-        :param data: List of Dicts containing trade data
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
         """
         raise NotImplementedError()
 
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> List[Dict]:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
@@ -140,9 +146,15 @@ class JsonDataHandler(IDataHandler):
         """
         filename = self._pair_trades_filename(self._datadir, pair)
         tradesdata = misc.file_load_json(filename)
+
         if not tradesdata:
             return []
 
+        if isinstance(tradesdata[0], dict):
+            # Convert trades dict to list
+            logger.info("Old trades format detected - converting")
+            tradesdata = trades_dict_to_list(tradesdata)
+            pass
         return tradesdata
 
     def trades_purge(self, pair: str) -> bool:

freqtrade/edge/edge_positioning.py

@@ -238,20 +238,9 @@ class Edge:
         :param result Dataframe
         :return: result Dataframe
         """
-
-        # stake and fees
-        # stake = 0.015
-        # 0.05% is 0.0005
-        # fee = 0.001
-
-        # we set stake amount to an arbitrary amount.
-        # as it doesn't change the calculation.
-        # all returned values are relative.
-        # they are defined as ratios.
+        # We set stake amount to an arbitrary amount, as it doesn't change the calculation.
+        # All returned values are relative, they are defined as ratios.
         stake = 0.015
-        fee = self.fee
-        open_fee = fee / 2
-        close_fee = fee / 2
 
         result['trade_duration'] = result['close_time'] - result['open_time']
 
@@ -262,12 +251,12 @@ class Edge:
 
         # Buy Price
         result['buy_vol'] = stake / result['open_rate']  # How many target are we buying
-        result['buy_fee'] = stake * open_fee
+        result['buy_fee'] = stake * self.fee
         result['buy_spend'] = stake + result['buy_fee']  # How much we're spending
 
         # Sell price
         result['sell_sum'] = result['buy_vol'] * result['close_rate']
-        result['sell_fee'] = result['sell_sum'] * close_fee
+        result['sell_fee'] = result['sell_sum'] * self.fee
         result['sell_take'] = result['sell_sum'] - result['sell_fee']
 
         # profit_ratio

freqtrade/exchange/binance.py

@@ -72,7 +72,7 @@ class Binance(Exchange):
             rate = self.price_to_precision(pair, rate)
 
             order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
-                                           amount=amount, price=stop_price, params=params)
+                                           amount=amount, price=rate, params=params)
             logger.info('stoploss limit order added for %s. '
                         'stop price: %s. limit: %s', pair, stop_price, rate)
             return order

freqtrade/exchange/exchange.py

@@ -18,12 +18,11 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE,
                                             TRUNCATE, decimal_to_precision)
 from pandas import DataFrame
 
-from freqtrade.data.converter import ohlcv_to_dataframe
+from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
 from freqtrade.exceptions import (DependencyException, InvalidOrderException,
                                   OperationalException, TemporaryError)
 from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
-from freqtrade.misc import deep_merge_dicts
-
+from freqtrade.misc import deep_merge_dicts, safe_value_fallback
 
 CcxtModuleType = Any
 
@@ -472,26 +471,31 @@ class Exchange:
             'pair': pair,
             'price': rate,
             'amount': _amount,
-            "cost": _amount * rate,
+            'cost': _amount * rate,
             'type': ordertype,
             'side': side,
             'remaining': _amount,
             'datetime': arrow.utcnow().isoformat(),
             'status': "closed" if ordertype == "market" else "open",
             'fee': None,
-            "info": {}
+            'info': {}
         }
-        self._store_dry_order(dry_order)
+        self._store_dry_order(dry_order, pair)
         # Copy order and close it - so the returned order is open unless it's a market order
         return dry_order
 
-    def _store_dry_order(self, dry_order: Dict) -> None:
+    def _store_dry_order(self, dry_order: Dict, pair: str) -> None:
         closed_order = dry_order.copy()
-        if closed_order["type"] in ["market", "limit"]:
+        if closed_order['type'] in ["market", "limit"]:
             closed_order.update({
-                "status": "closed",
-                "filled": closed_order["amount"],
-                "remaining": 0
+                'status': 'closed',
+                'filled': closed_order['amount'],
+                'remaining': 0,
+                'fee': {
+                    'currency': self.get_pair_quote_currency(pair),
+                    'cost': dry_order['cost'] * self.get_fee(pair),
+                    'rate': self.get_fee(pair)
+                }
             })
         if closed_order["type"] in ["stop_loss_limit"]:
             closed_order["info"].update({"stopPrice": closed_order["price"]})
@@ -769,7 +773,7 @@ class Exchange:
     @retrier_async
     async def _async_fetch_trades(self, pair: str,
                                   since: Optional[int] = None,
-                                  params: Optional[dict] = None) -> List[Dict]:
+                                  params: Optional[dict] = None) -> List[List]:
         """
         Asyncronously gets trade history using fetch_trades.
         Handles exchange errors, does one call to the exchange.
@@ -789,7 +793,7 @@ class Exchange:
                     '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else ''
                 )
                 trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
-            return trades
+            return trades_dict_to_list(trades)
         except ccxt.NotSupported as e:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching historical trade data.'
@@ -803,7 +807,7 @@ class Exchange:
     async def _async_get_trade_history_id(self, pair: str,
                                           until: int,
                                           since: Optional[int] = None,
-                                          from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
+                                          from_id: Optional[str] = None) -> Tuple[str, List[List]]:
         """
         Asyncronously gets trade history using fetch_trades
         use this when exchange uses id-based iteration (check `self._trades_pagination`)
@@ -814,7 +818,7 @@ class Exchange:
         returns tuple: (pair, trades-list)
         """
 
-        trades: List[Dict] = []
+        trades: List[List] = []
 
         if not from_id:
             # Fetch first elements using timebased method to get an ID to paginate on
@@ -823,7 +827,9 @@ class Exchange:
             # e.g. Binance returns the "last 1000" candles within a 1h time interval
             # - so we will miss the first trades.
             t = await self._async_fetch_trades(pair, since=since)
-            from_id = t[-1]['id']
+            # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+            # DEFAULT_TRADES_COLUMNS: 1 -> id
+            from_id = t[-1][1]
             trades.extend(t[:-1])
         while True:
             t = await self._async_fetch_trades(pair,
@@ -831,21 +837,21 @@ class Exchange:
             if len(t):
                 # Skip last id since its the key for the next call
                 trades.extend(t[:-1])
-                if from_id == t[-1]['id'] or t[-1]['timestamp'] > until:
+                if from_id == t[-1][1] or t[-1][0] > until:
                     logger.debug(f"Stopping because from_id did not change. "
-                                 f"Reached {t[-1]['timestamp']} > {until}")
+                                 f"Reached {t[-1][0]} > {until}")
                     # Reached the end of the defined-download period - add last trade as well.
                     trades.extend(t[-1:])
                     break
 
-                from_id = t[-1]['id']
+                from_id = t[-1][1]
             else:
                 break
 
         return (pair, trades)
 
     async def _async_get_trade_history_time(self, pair: str, until: int,
-                                            since: Optional[int] = None) -> Tuple[str, List]:
+                                            since: Optional[int] = None) -> Tuple[str, List[List]]:
         """
         Asyncronously gets trade history using fetch_trades,
         when the exchange uses time-based iteration (check `self._trades_pagination`)
@@ -855,16 +861,18 @@ class Exchange:
         returns tuple: (pair, trades-list)
         """
 
-        trades: List[Dict] = []
+        trades: List[List] = []
+        # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+        # DEFAULT_TRADES_COLUMNS: 1 -> id
         while True:
             t = await self._async_fetch_trades(pair, since=since)
             if len(t):
-                since = t[-1]['timestamp']
+                since = t[-1][1]
                 trades.extend(t)
                 # Reached the end of the defined-download period
-                if until and t[-1]['timestamp'] > until:
+                if until and t[-1][0] > until:
                     logger.debug(
-                        f"Stopping because until was reached. {t[-1]['timestamp']} > {until}")
+                        f"Stopping because until was reached. {t[-1][0]} > {until}")
                     break
             else:
                 break
@@ -874,7 +882,7 @@ class Exchange:
     async def _async_get_trade_history(self, pair: str,
                                        since: Optional[int] = None,
                                        until: Optional[int] = None,
-                                       from_id: Optional[str] = None) -> Tuple[str, List[Dict]]:
+                                       from_id: Optional[str] = None) -> Tuple[str, List[List]]:
         """
         Async wrapper handling downloading trades using either time or id based methods.
         """
@@ -1041,9 +1049,9 @@ class Exchange:
 
             return matched_trades
 
-        except ccxt.NetworkError as e:
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
             raise TemporaryError(
-                f'Could not get trades due to networking error. Message: {e}') from e
+                f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
@@ -1063,6 +1071,61 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    @staticmethod
+    def order_has_fee(order: Dict) -> bool:
+        """
+        Verifies if the passed in order dict has the needed keys to extract fees,
+        and that these keys (currency, cost) are not empty.
+        :param order: Order or trade (one trade) dict
+        :return: True if the fee substructure contains currency and cost, false otherwise
+        """
+        if not isinstance(order, dict):
+            return False
+        return ('fee' in order and order['fee'] is not None
+                and (order['fee'].keys() >= {'currency', 'cost'})
+                and order['fee']['currency'] is not None
+                and order['fee']['cost'] is not None
+                )
+
+    def calculate_fee_rate(self, order: Dict) -> Optional[float]:
+        """
+        Calculate fee rate if it's not given by the exchange.
+        :param order: Order or trade (one trade) dict
+        """
+        if order['fee'].get('rate') is not None:
+            return order['fee'].get('rate')
+        fee_curr = order['fee']['currency']
+        # Calculate fee based on order details
+        if fee_curr in self.get_pair_base_currency(order['symbol']):
+            # Base currency - divide by amount
+            return round(
+                order['fee']['cost'] / safe_value_fallback(order, order, 'filled', 'amount'), 8)
+        elif fee_curr in self.get_pair_quote_currency(order['symbol']):
+            # Quote currency - divide by cost
+            return round(order['fee']['cost'] / order['cost'], 8)
+        else:
+            # If Fee currency is a different currency
+            try:
+                comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency'])
+                tick = self.fetch_ticker(comb)
+
+                fee_to_quote_rate = safe_value_fallback(tick, tick, 'last', 'ask')
+                return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8)
+            except DependencyException:
+                return None
+
+    def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]:
+        """
+        Extract tuple of cost, currency, rate.
+        Requires order_has_fee to run first!
+        :param order: Order or trade (one trade) dict
+        :return: Tuple with cost, currency, rate of the given fee dict
+        """
+        return (order['fee']['cost'],
+                order['fee']['currency'],
+                self.calculate_fee_rate(order))
+        # calculate rate ? (order['fee']['cost'] / (order['amount'] * order['price']))
+
 
 def is_exchange_bad(exchange_name: str) -> bool:
     return exchange_name in BAD_EXCHANGES

freqtrade/freqtradebot.py

@@ -54,8 +54,11 @@ class FreqtradeBot:
         # Init objects
         self.config = config
 
-        self._sell_rate_cache = TTLCache(maxsize=100, ttl=5)
-        self._buy_rate_cache = TTLCache(maxsize=100, ttl=5)
+        # Cache values for 1800 to avoid frequent polling of the exchange for prices
+        # Caching only applies to RPC methods, so prices for open trades are still
+        # refreshed once every iteration.
+        self._sell_rate_cache = TTLCache(maxsize=100, ttl=1800)
+        self._buy_rate_cache = TTLCache(maxsize=100, ttl=1800)
 
         self.strategy: IStrategy = StrategyResolver.load_strategy(self.config)
 
@@ -68,15 +71,15 @@ class FreqtradeBot:
 
         self.wallets = Wallets(self.config, self.exchange)
 
-        self.dataprovider = DataProvider(self.config, self.exchange)
+        self.pairlists = PairListManager(self.exchange, self.config)
+
+        self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists)
 
         # Attach Dataprovider to Strategy baseclass
         IStrategy.dp = self.dataprovider
         # Attach Wallets to Strategy baseclass
         IStrategy.wallets = self.wallets
 
-        self.pairlists = PairListManager(self.exchange, self.config)
-
         # Initializing Edge only if enabled
         self.edge = Edge(self.config, self.exchange, self.strategy) if \
             self.config.get('edge', {}).get('enabled', False) else None
@@ -620,7 +623,7 @@ class FreqtradeBot:
                     trades_closed += 1
                     continue
                 # Check if we can sell our current pair
-                if trade.open_order_id is None and self.handle_trade(trade):
+                if trade.open_order_id is None and trade.is_open and self.handle_trade(trade):
                     trades_closed += 1
 
             except DependencyException as exception:
@@ -762,7 +765,7 @@ class FreqtradeBot:
         # We check if stoploss order is fulfilled
         if stoploss_order and stoploss_order['status'] == 'closed':
             trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value
-            trade.update(stoploss_order)
+            self.update_trade_state(trade, stoploss_order, sl_order=True)
             # Lock pair for one candle to prevent immediate rebuys
             self.strategy.lock_pair(trade.pair,
                                     timeframe_to_next_date(self.config['ticker_interval']))
@@ -922,7 +925,8 @@ class FreqtradeBot:
         """
         was_trade_fully_canceled = False
 
-        if order['status'] != 'canceled':
+        # Cancelled orders may have the status of 'canceled' or 'closed'
+        if order['status'] not in ('canceled', 'closed'):
             reason = constants.CANCEL_REASON['TIMEOUT']
             corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
                                                             trade.amount)
@@ -933,7 +937,10 @@ class FreqtradeBot:
 
         logger.info('Buy order %s for %s.', reason, trade)
 
-        if safe_value_fallback(corder, order, 'remaining', 'remaining') == order['amount']:
+        # Using filled to determine the filled amount
+        filled_amount = safe_value_fallback(corder, order, 'filled', 'filled')
+
+        if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
             logger.info('Buy order fully cancelled. Removing %s from database.', trade)
             # if trade is not partially completed, just delete the trade
             Trade.session.delete(trade)
@@ -945,8 +952,7 @@ class FreqtradeBot:
             # cancel_order may not contain the full order dict, so we need to fallback
             # to the order dict aquired before cancelling.
             # we need to fall back to the values from order if corder does not contain these keys.
-            trade.amount = order['amount'] - safe_value_fallback(corder, order,
-                                                                 'remaining', 'remaining')
+            trade.amount = filled_amount
             trade.stake_amount = trade.amount * trade.open_rate
             self.update_trade_state(trade, corder, trade.amount)
 
@@ -966,11 +972,15 @@ class FreqtradeBot:
         Sell cancel - cancel order and update trade
         :return: Reason for cancel
         """
-        # if trade is not partially completed, just cancel the trade
+        # if trade is not partially completed, just cancel the order
         if order['remaining'] == order['amount'] or order.get('filled') == 0.0:
             if not self.exchange.check_order_canceled_empty(order):
-                # if trade is not partially completed, just delete the trade
+                try:
+                    # if trade is not partially completed, just delete the order
                     self.exchange.cancel_order(trade.open_order_id, trade.pair)
+                except InvalidOrderException:
+                    logger.exception(f"Could not cancel sell order {trade.open_order_id}")
+                    return 'error cancelling order'
                 logger.info('Sell order %s for %s.', reason, trade)
             else:
                 reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
@@ -1014,7 +1024,7 @@ class FreqtradeBot:
         if wallet_amount >= amount:
             return amount
         elif wallet_amount > amount * 0.98:
-            logger.info(f"{pair} - Falling back to wallet-amount.")
+            logger.info(f"{pair} - Falling back to wallet-amount {wallet_amount} -> {amount}.")
             return wallet_amount
         else:
             raise DependencyException(
@@ -1064,7 +1074,7 @@ class FreqtradeBot:
         trade.sell_reason = sell_reason.value
         # In case of market sell orders the order can be closed immediately
         if order.get('status', 'unknown') == 'closed':
-            trade.update(order)
+            self.update_trade_state(trade, order)
         Trade.session.flush()
 
         # Lock pair for one candle to prevent immediate rebuys
@@ -1155,7 +1165,7 @@ class FreqtradeBot:
 #
 
     def update_trade_state(self, trade: Trade, action_order: dict = None,
-                           order_amount: float = None) -> bool:
+                           order_amount: float = None, sl_order: bool = False) -> bool:
         """
         Checks trades with open orders and updates the amount if necessary
         Handles closing both buy and sell orders.
@@ -1163,12 +1173,17 @@ class FreqtradeBot:
         """
         # Get order details for actual price per unit
         if trade.open_order_id:
+            order_id = trade.open_order_id
+        elif trade.stoploss_order_id and sl_order:
+            order_id = trade.stoploss_order_id
+        else:
+            return False
         # Update trade with order values
         logger.info('Found open order for %s', trade)
         try:
-                order = action_order or self.exchange.get_order(trade.open_order_id, trade.pair)
+            order = action_order or self.exchange.get_order(order_id, trade.pair)
         except InvalidOrderException as exception:
-                logger.warning('Unable to fetch order %s: %s', trade.open_order_id, exception)
+            logger.warning('Unable to fetch order %s: %s', order_id, exception)
             return False
         # Try update amount (binance-fix)
         try:
@@ -1176,8 +1191,6 @@ class FreqtradeBot:
             if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
                 order['amount'] = new_amount
                 order.pop('filled', None)
-                    # Fee was applied, so set to 0
-                    trade.fee_open = 0
                 trade.recalc_open_trade_price()
         except DependencyException as exception:
             logger.warning("Could not update trade amount: %s", exception)
@@ -1191,56 +1204,94 @@ class FreqtradeBot:
         # Updating wallets when order is closed
         if not trade.is_open:
             self.wallets.update()
-
         return False
 
+    def apply_fee_conditional(self, trade: Trade, trade_base_currency: str,
+                              amount: float, fee_abs: float) -> float:
+        """
+        Applies the fee to amount (either from Order or from Trades).
+        Can eat into dust if more than the required asset is available.
+        """
+        self.wallets.update()
+        if fee_abs != 0 and self.wallets.get_free(trade_base_currency) >= amount:
+            # Eat into dust if we own more than base currency
+            logger.info(f"Fee amount for {trade} was in base currency - "
+                        f"Eating Fee {fee_abs} into dust.")
+        elif fee_abs != 0:
+            real_amount = self.exchange.amount_to_precision(trade.pair, amount - fee_abs)
+            logger.info(f"Applying fee on amount for {trade} "
+                        f"(from {amount} to {real_amount}).")
+            return real_amount
+        return amount
+
     def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float:
         """
-        Get real amount for the trade
+        Detect and update trade fee.
+        Calls trade.update_fee() uppon correct detection.
+        Returns modified amount if the fee was taken from the destination currency.
         Necessary for exchanges which charge fees in base currency (e.g. binance)
+        :return: identical (or new) amount for the trade
         """
+        # Init variables
         if order_amount is None:
             order_amount = order['amount']
         # Only run for closed orders
-        if trade.fee_open == 0 or order['status'] == 'open':
+        if trade.fee_updated(order.get('side', '')) or order['status'] == 'open':
             return order_amount
 
         trade_base_currency = self.exchange.get_pair_base_currency(trade.pair)
         # use fee from order-dict if possible
-        if ('fee' in order and order['fee'] is not None and
-                (order['fee'].keys() >= {'currency', 'cost'})):
-            if (order['fee']['currency'] is not None and
-                    order['fee']['cost'] is not None and
-                    trade_base_currency == order['fee']['currency']):
-                new_amount = order_amount - order['fee']['cost']
-                logger.info("Applying fee on amount for %s (from %s to %s) from Order",
-                            trade, order['amount'], new_amount)
-                return new_amount
+        if self.exchange.order_has_fee(order):
+            fee_cost, fee_currency, fee_rate = self.exchange.extract_cost_curr_rate(order)
+            logger.info(f"Fee for Trade {trade} [{order.get('side')}]: "
+                        f"{fee_cost:.8g} {fee_currency} - rate: {fee_rate}")
 
-        # Fallback to Trades
+            trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
+            if trade_base_currency == fee_currency:
+                # Apply fee to amount
+                return self.apply_fee_conditional(trade, trade_base_currency,
+                                                  amount=order_amount, fee_abs=fee_cost)
+            return order_amount
+        return self.fee_detection_from_trades(trade, order, order_amount)
+
+    def fee_detection_from_trades(self, trade: Trade, order: Dict, order_amount: float) -> float:
+        """
+        fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee.
+        """
         trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair,
                                                     trade.open_date)
 
         if len(trades) == 0:
             logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade)
             return order_amount
+        fee_currency = None
         amount = 0
-        fee_abs = 0
+        fee_abs = 0.0
+        fee_cost = 0.0
+        trade_base_currency = self.exchange.get_pair_base_currency(trade.pair)
+        fee_rate_array: List[float] = []
         for exectrade in trades:
             amount += exectrade['amount']
-            if ("fee" in exectrade and exectrade['fee'] is not None and
-                    (exectrade['fee'].keys() >= {'currency', 'cost'})):
+            if self.exchange.order_has_fee(exectrade):
+                fee_cost_, fee_currency, fee_rate_ = self.exchange.extract_cost_curr_rate(exectrade)
+                fee_cost += fee_cost_
+                if fee_rate_ is not None:
+                    fee_rate_array.append(fee_rate_)
                 # only applies if fee is in quote currency!
-                if (exectrade['fee']['currency'] is not None and
-                        exectrade['fee']['cost'] is not None and
-                        trade_base_currency == exectrade['fee']['currency']):
-                    fee_abs += exectrade['fee']['cost']
+                if trade_base_currency == fee_currency:
+                    fee_abs += fee_cost_
+        # Ensure at least one trade was found:
+        if fee_currency:
+            # fee_rate should use mean
+            fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None
+            trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
 
         if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC):
             logger.warning(f"Amount {amount} does not match amount {trade.amount}")
             raise DependencyException("Half bought? Amounts don't match")
-        real_amount = amount - fee_abs
+
         if fee_abs != 0:
-            logger.info(f"Applying fee on amount for {trade} "
-                        f"(from {order_amount} to {real_amount}) from Trades")
-        return real_amount
+            return self.apply_fee_conditional(trade, trade_base_currency,
+                                              amount=amount, fee_abs=fee_abs)
+        else:
+            return amount

freqtrade/optimize/hyperopt.py

@@ -387,12 +387,19 @@ class Hyperopt:
         trials = json_normalize(results, max_level=1)
         trials['Best'] = ''
         trials['Stake currency'] = config['stake_currency']
-        trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
+
+        base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
                         'results_metrics.avg_profit', 'results_metrics.total_profit',
                         'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
-                         'loss', 'is_initial_point', 'is_best']]
-        trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency',
+                        'loss', 'is_initial_point', 'is_best']
+        param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
+        trials = trials[base_metrics + param_metrics]
+
+        base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency',
                         'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best']
+        param_columns = list(results[0]['params_dict'].keys())
+        trials.columns = base_columns + param_columns
+
         trials['is_profit'] = False
         trials.loc[trials['is_initial_point'], 'Best'] = '*'
         trials.loc[trials['is_best'], 'Best'] = 'Best'

freqtrade/persistence.py

@@ -86,11 +86,15 @@ def check_migrate(engine) -> None:
         logger.debug(f'trying {table_back_name}')
 
     # Check for latest column
-    if not has_column(cols, 'close_profit_abs'):
+    if not has_column(cols, 'fee_close_cost'):
         logger.info(f'Running database migration - backup available as {table_back_name}')
 
         fee_open = get_column_def(cols, 'fee_open', 'fee')
+        fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
+        fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
         fee_close = get_column_def(cols, 'fee_close', 'fee')
+        fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
+        fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
         open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
         close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
         stop_loss = get_column_def(cols, 'stop_loss', '0.0')
@@ -120,7 +124,9 @@ def check_migrate(engine) -> None:
 
         # Copy data back - following the correct schema
         engine.execute(f"""insert into trades
-                (id, exchange, pair, is_open, fee_open, fee_close, open_rate,
+                (id, exchange, pair, is_open,
+                fee_open, fee_open_cost, fee_open_currency,
+                fee_close, fee_close_cost, fee_open_currency, open_rate,
                 open_rate_requested, close_rate, close_rate_requested, close_profit,
                 stake_amount, amount, open_date, close_date, open_order_id,
                 stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
@@ -136,7 +142,9 @@ def check_migrate(engine) -> None:
                     else pair
                     end
                 pair,
-                is_open, {fee_open} fee_open, {fee_close} fee_close,
+                is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
+                {fee_open_currency} fee_open_currency, {fee_close} fee_close,
+                {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
                 open_rate, {open_rate_requested} open_rate_requested, close_rate,
                 {close_rate_requested} close_rate_requested, close_profit,
                 stake_amount, amount, open_date, close_date, open_order_id,
@@ -185,7 +193,11 @@ class Trade(_DECL_BASE):
     pair = Column(String, nullable=False, index=True)
     is_open = Column(Boolean, nullable=False, default=True, index=True)
     fee_open = Column(Float, nullable=False, default=0.0)
+    fee_open_cost = Column(Float, nullable=True)
+    fee_open_currency = Column(String, nullable=True)
     fee_close = Column(Float, nullable=False, default=0.0)
+    fee_close_cost = Column(Float, nullable=True)
+    fee_close_currency = Column(String, nullable=True)
     open_rate = Column(Float)
     open_rate_requested = Column(Float)
     # open_trade_price - calculated via _calc_open_trade_price
@@ -235,7 +247,11 @@ class Trade(_DECL_BASE):
             'pair': self.pair,
             'is_open': self.is_open,
             'fee_open': self.fee_open,
+            'fee_open_cost': self.fee_open_cost,
+            'fee_open_currency': self.fee_open_currency,
             'fee_close': self.fee_close,
+            'fee_close_cost': self.fee_close_cost,
+            'fee_close_currency': self.fee_close_currency,
             'open_date_hum': arrow.get(self.open_date).humanize(),
             'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"),
             'close_date_hum': (arrow.get(self.close_date).humanize()
@@ -360,6 +376,35 @@ class Trade(_DECL_BASE):
             self
         )
 
+    def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float],
+                   side: str) -> None:
+        """
+        Update Fee parameters. Only acts once per side
+        """
+        if side == 'buy' and self.fee_open_currency is None:
+            self.fee_open_cost = fee_cost
+            self.fee_open_currency = fee_currency
+            if fee_rate is not None:
+                self.fee_open = fee_rate
+                # Assume close-fee will fall into the same fee category and take an educated guess
+                self.fee_close = fee_rate
+        elif side == 'sell' and self.fee_close_currency is None:
+            self.fee_close_cost = fee_cost
+            self.fee_close_currency = fee_currency
+            if fee_rate is not None:
+                self.fee_close = fee_rate
+
+    def fee_updated(self, side: str) -> bool:
+        """
+        Verify if this side (buy / sell) has already been updated
+        """
+        if side == 'buy':
+            return self.fee_open_currency is not None
+        elif side == 'sell':
+            return self.fee_close_currency is not None
+        else:
+            return False
+
     def _calc_open_trade_price(self) -> float:
         """
         Calculate the open_rate including open_fee.

freqtrade/rpc/api_server.py

@@ -2,11 +2,17 @@ import logging
 import threading
 from datetime import date, datetime
 from ipaddress import IPv4Address
-from typing import Dict, Callable, Any
+from typing import Any, Callable, Dict
 
 from arrow import Arrow
 from flask import Flask, jsonify, request
 from flask.json import JSONEncoder
+from flask_cors import CORS
+from flask_jwt_extended import (JWTManager, create_access_token,
+                                create_refresh_token, get_jwt_identity,
+                                jwt_refresh_token_required,
+                                verify_jwt_in_request_optional)
+from werkzeug.security import safe_str_cmp
 from werkzeug.serving import make_server
 
 from freqtrade.__init__ import __version__
@@ -38,9 +44,9 @@ class ArrowJSONEncoder(JSONEncoder):
 def require_login(func: Callable[[Any, Any], Any]):
 
     def func_wrapper(obj, *args, **kwargs):
-
+        verify_jwt_in_request_optional()
         auth = request.authorization
-        if auth and obj.check_auth(auth.username, auth.password):
+        if get_jwt_identity() or auth and obj.check_auth(auth.username, auth.password):
             return func(obj, *args, **kwargs)
         else:
             return jsonify({"error": "Unauthorized"}), 401
@@ -70,8 +76,8 @@ class ApiServer(RPC):
     """
 
     def check_auth(self, username, password):
-        return (username == self._config['api_server'].get('username') and
-                password == self._config['api_server'].get('password'))
+        return (safe_str_cmp(username, self._config['api_server'].get('username')) and
+                safe_str_cmp(password, self._config['api_server'].get('password')))
 
     def __init__(self, freqtrade) -> None:
         """
@@ -83,6 +89,13 @@ class ApiServer(RPC):
 
         self._config = freqtrade.config
         self.app = Flask(__name__)
+        self._cors = CORS(self.app, resources={r"/api/*": {"origins": "*"}})
+
+        # Setup the Flask-JWT-Extended extension
+        self.app.config['JWT_SECRET_KEY'] = self._config['api_server'].get(
+            'jwt_secret_key', 'super-secret')
+
+        self.jwt = JWTManager(self.app)
         self.app.json_encoder = ArrowJSONEncoder
 
         # Register application handling
@@ -148,6 +161,10 @@ class ApiServer(RPC):
         self.app.register_error_handler(404, self.page_not_found)
 
         # Actions to control the bot
+        self.app.add_url_rule(f'{BASE_URI}/token/login', 'login',
+                              view_func=self._token_login, methods=['POST'])
+        self.app.add_url_rule(f'{BASE_URI}/token/refresh', 'token_refresh',
+                              view_func=self._token_refresh, methods=['POST'])
         self.app.add_url_rule(f'{BASE_URI}/start', 'start',
                               view_func=self._start, methods=['POST'])
         self.app.add_url_rule(f'{BASE_URI}/stop', 'stop', view_func=self._stop, methods=['POST'])
@@ -199,6 +216,37 @@ class ApiServer(RPC):
             'code': 404
         }), 404
 
+    @require_login
+    @rpc_catch_errors
+    def _token_login(self):
+        """
+        Handler for /token/login
+        Returns a JWT token
+        """
+        auth = request.authorization
+        if auth and self.check_auth(auth.username, auth.password):
+            keystuff = {'u': auth.username}
+            ret = {
+                'access_token': create_access_token(identity=keystuff),
+                'refresh_token': create_refresh_token(identity=keystuff),
+            }
+            return self.rest_dump(ret)
+
+        return jsonify({"error": "Unauthorized"}), 401
+
+    @jwt_refresh_token_required
+    @rpc_catch_errors
+    def _token_refresh(self):
+        """
+        Handler for /token/refresh
+        Returns a JWT token based on a JWT refresh token
+        """
+        current_user = get_jwt_identity()
+        new_token = create_access_token(identity=current_user, fresh=False)
+
+        ret = {'access_token': new_token}
+        return self.rest_dump(ret)
+
     @require_login
     @rpc_catch_errors
     def _start(self):

freqtrade/rpc/rpc.py

@@ -94,6 +94,7 @@ class RPC:
             'dry_run': config['dry_run'],
             'stake_currency': config['stake_currency'],
             'stake_amount': config['stake_amount'],
+            'max_open_trades': config['max_open_trades'],
             'minimal_roi': config['minimal_roi'].copy(),
             'stoploss': config['stoploss'],
             'trailing_stop': config['trailing_stop'],
@@ -103,6 +104,8 @@ class RPC:
             'ticker_interval': config['ticker_interval'],
             'exchange': config['exchange']['name'],
             'strategy': config['strategy'],
+            'forcebuy_enabled': config.get('forcebuy_enable', False),
+            'state': str(self._freqtrade.state)
         }
         return val
 

freqtrade/rpc/telegram.py

@@ -579,7 +579,7 @@ class Telegram(RPC):
                   "*/whitelist:* `Show current whitelist` \n" \
                   "*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs " \
                   "to the blacklist.` \n" \
-                  "*/edge:* `Shows validated pairs by Edge if it is enabeld` \n" \
+                  "*/edge:* `Shows validated pairs by Edge if it is enabled` \n" \
                   "*/help:* `This help message`\n" \
                   "*/version:* `Show version`"
 
@@ -621,10 +621,12 @@ class Telegram(RPC):
             f"*Mode:* `{'Dry-run' if val['dry_run'] else 'Live'}`\n"
             f"*Exchange:* `{val['exchange']}`\n"
             f"*Stake per trade:* `{val['stake_amount']} {val['stake_currency']}`\n"
+            f"*Max open Trades:* `{val['max_open_trades']}`\n"
             f"*Minimum ROI:* `{val['minimal_roi']}`\n"
             f"{sl_info}"
             f"*Ticker Interval:* `{val['ticker_interval']}`\n"
-            f"*Strategy:* `{val['strategy']}`"
+            f"*Strategy:* `{val['strategy']}`\n"
+            f"*Current state:* `{val['state']}`"
         )
 
     def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None:

freqtrade/state.py

@@ -14,6 +14,9 @@ class State(Enum):
     STOPPED = 2
     RELOAD_CONF = 3
 
+    def __str__(self):
+        return f"{self.name.lower()}"
+
 
 class RunMode(Enum):
     """

requirements-common.txt

@@ -1,15 +1,15 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.27.1
+ccxt==1.27.49
 SQLAlchemy==1.3.16
-python-telegram-bot==12.6.1
-arrow==0.15.5
+python-telegram-bot==12.7
+arrow==0.15.6
 cachetools==4.1.0
 requests==2.23.0
 urllib3==1.25.9
 wrapt==1.12.1
 jsonschema==3.2.0
-TA-Lib==0.4.17
+TA-Lib==0.4.18
 tabulate==0.8.7
 pycoingecko==1.2.0
 jinja2==2.11.2
@@ -25,6 +25,8 @@ sdnotify==0.3.2
 
 # Api server
 flask==1.1.2
+flask-jwt-extended==3.24.1
+flask-cors==3.0.8
 
 # Support for colorized terminal output
 colorama==0.4.3

requirements-dev.txt

@@ -8,8 +8,8 @@ flake8==3.7.9
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==4.1.0
 mypy==0.770
-pytest==5.4.1
-pytest-asyncio==0.11.0
+pytest==5.4.2
+pytest-asyncio==0.12.0
 pytest-cov==2.8.1
 pytest-mock==3.1.0
 pytest-random-order==1.0.4

requirements-hyperopt.txt

@@ -7,4 +7,4 @@ scikit-learn==0.22.2.post1
 scikit-optimize==0.7.4
 filelock==3.0.12
 joblib==0.14.1
-progressbar2==3.51.0
+progressbar2==3.51.3

requirements-plot.txt

@@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==4.6.0
+plotly==4.7.1
 

requirements.txt

@@ -1,5 +1,5 @@
 # Load common requirements
 -r requirements-common.txt
 
-numpy==1.18.3
+numpy==1.18.4
 pandas==1.0.3

setup.py

@@ -16,7 +16,7 @@ if readme_file.is_file():
     readme_long = (Path(__file__).parent / "README.md").read_text()
 
 # Requirements used for submodules
-api = ['flask']
+api = ['flask', 'flask-jwt-extended', 'flask-cors']
 plot = ['plotly>=4.0']
 hyperopt = [
     'scipy',

tests/commands/test_commands.py

@@ -10,11 +10,13 @@ from freqtrade.commands import (start_convert_data, start_create_userdir,
                                 start_list_hyperopts, start_list_markets,
                                 start_list_strategies, start_list_timeframes,
                                 start_new_hyperopt, start_new_strategy,
-                                start_test_pairlist, start_trading)
+                                start_show_trades, start_test_pairlist,
+                                start_trading)
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.exceptions import OperationalException
 from freqtrade.state import RunMode
-from tests.conftest import (get_args, log_has, log_has_re, patch_exchange,
+from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re,
+                            patch_exchange,
                             patched_configuration_load_config_file)
 
 
@@ -30,7 +32,7 @@ def test_setup_utils_configuration():
     assert config['exchange']['secret'] == ''
 
 
-def test_start_trading_fail(mocker):
+def test_start_trading_fail(mocker, caplog):
 
     mocker.patch("freqtrade.worker.Worker.run", MagicMock(side_effect=OperationalException))
 
@@ -41,16 +43,15 @@ def test_start_trading_fail(mocker):
         'trade',
         '-c', 'config.json.example'
     ]
-    with pytest.raises(OperationalException):
     start_trading(get_args(args))
     assert exitmock.call_count == 1
 
     exitmock.reset_mock()
-
+    caplog.clear()
     mocker.patch("freqtrade.worker.Worker.__init__", MagicMock(side_effect=OperationalException))
-    with pytest.raises(OperationalException):
     start_trading(get_args(args))
     assert exitmock.call_count == 0
+    assert log_has('Fatal exception!', caplog)
 
 
 def test_list_exchanges(capsys):
@@ -1040,3 +1041,46 @@ def test_convert_data_trades(mocker, testdatadir):
     assert trades_mock.call_args[1]['convert_from'] == 'jsongz'
     assert trades_mock.call_args[1]['convert_to'] == 'json'
     assert trades_mock.call_args[1]['erase'] is False
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_show_trades(mocker, fee, capsys, caplog):
+    mocker.patch("freqtrade.persistence.init")
+    create_mock_trades(fee)
+    args = [
+        "show-trades",
+        "--db-url",
+        "sqlite:///"
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_show_trades(pargs)
+    assert log_has("Printing 3 Trades: ", caplog)
+    captured = capsys.readouterr()
+    assert "Trade(id=1" in captured.out
+    assert "Trade(id=2" in captured.out
+    assert "Trade(id=3" in captured.out
+    args = [
+        "show-trades",
+        "--db-url",
+        "sqlite:///",
+        "--print-json",
+        "--trade-ids", "1", "2"
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_show_trades(pargs)
+
+    captured = capsys.readouterr()
+    assert log_has("Printing 2 Trades: ", caplog)
+    assert '"trade_id": 1' in captured.out
+    assert '"trade_id": 2' in captured.out
+    assert '"trade_id": 3' not in captured.out
+    args = [
+        "show-trades",
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+
+    with pytest.raises(OperationalException, match=r"--db-url is required for this command."):
+        start_show_trades(pargs)

tests/conftest.py

@@ -305,7 +305,8 @@ def default_conf(testdatadir):
         "user_data_dir": Path("user_data"),
         "verbosity": 3,
         "strategy_path": str(Path(__file__).parent / "strategy" / "strats"),
-        "strategy": "DefaultStrategy"
+        "strategy": "DefaultStrategy",
+        "internals": {},
     }
     return configuration
 
@@ -780,7 +781,7 @@ def limit_buy_order():
         'id': 'mocked_limit_buy',
         'type': 'limit',
         'side': 'buy',
-        'pair': 'mocked',
+        'symbol': 'mocked',
         'datetime': arrow.utcnow().isoformat(),
         'price': 0.00001099,
         'amount': 90.99181073,
@@ -796,7 +797,7 @@ def market_buy_order():
         'id': 'mocked_market_buy',
         'type': 'market',
         'side': 'buy',
-        'pair': 'mocked',
+        'symbol': 'mocked',
         'datetime': arrow.utcnow().isoformat(),
         'price': 0.00004099,
         'amount': 91.99181073,
@@ -812,7 +813,7 @@ def market_sell_order():
         'id': 'mocked_limit_sell',
         'type': 'market',
         'side': 'sell',
-        'pair': 'mocked',
+        'symbol': 'mocked',
         'datetime': arrow.utcnow().isoformat(),
         'price': 0.00004173,
         'amount': 91.99181073,
@@ -828,7 +829,7 @@ def limit_buy_order_old():
         'id': 'mocked_limit_buy_old',
         'type': 'limit',
         'side': 'buy',
-        'pair': 'mocked',
+        'symbol': 'mocked',
         'datetime': str(arrow.utcnow().shift(minutes=-601).datetime),
         'price': 0.00001099,
         'amount': 90.99181073,
@@ -844,7 +845,7 @@ def limit_sell_order_old():
         'id': 'mocked_limit_sell_old',
         'type': 'limit',
         'side': 'sell',
-        'pair': 'ETH/BTC',
+        'symbol': 'ETH/BTC',
         'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
         'price': 0.00001099,
         'amount': 90.99181073,
@@ -860,7 +861,7 @@ def limit_buy_order_old_partial():
         'id': 'mocked_limit_buy_old_partial',
         'type': 'limit',
         'side': 'buy',
-        'pair': 'ETH/BTC',
+        'symbol': 'ETH/BTC',
         'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
         'price': 0.00001099,
         'amount': 90.99181073,
@@ -874,10 +875,103 @@ def limit_buy_order_old_partial():
 def limit_buy_order_old_partial_canceled(limit_buy_order_old_partial):
     res = deepcopy(limit_buy_order_old_partial)
     res['status'] = 'canceled'
-    res['fee'] = {'cost': 0.0001, 'currency': 'ETH'}
+    res['fee'] = {'cost': 0.023, 'currency': 'ETH'}
     return res
 
 
+@pytest.fixture(scope='function')
+def limit_buy_order_canceled_empty(request):
+    # Indirect fixture
+    # Documentation:
+    # https://docs.pytest.org/en/latest/example/parametrize.html#apply-indirect-on-particular-arguments
+
+    exchange_name = request.param
+    if exchange_name == 'ftx':
+        return {
+            'info': {},
+            'id': '1234512345',
+            'clientOrderId': None,
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 34.3225,
+            'amount': 0.55,
+            'cost': 0.0,
+            'average': None,
+            'filled': 0.0,
+            'remaining': 0.0,
+            'status': 'closed',
+            'fee': None,
+            'trades': None
+        }
+    elif exchange_name == 'kraken':
+        return {
+            'info': {},
+            'id': 'AZNPFF-4AC4N-7MKTAT',
+            'clientOrderId': None,
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'status': 'canceled',
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 34.3225,
+            'cost': 0.0,
+            'amount': 0.55,
+            'filled': 0.0,
+            'average': 0.0,
+            'remaining': 0.55,
+            'fee': {'cost': 0.0, 'rate': None, 'currency': 'USDT'},
+            'trades': []
+        }
+    elif exchange_name == 'binance':
+        return {
+            'info': {},
+            'id': '1234512345',
+            'clientOrderId': 'alb1234123',
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 0.016804,
+            'amount': 0.55,
+            'cost': 0.0,
+            'average': None,
+            'filled': 0.0,
+            'remaining': 0.55,
+            'status': 'canceled',
+            'fee': None,
+            'trades': None
+        }
+    else:
+        return {
+            'info': {},
+            'id': '1234512345',
+            'clientOrderId': 'alb1234123',
+            'timestamp': arrow.utcnow().shift(minutes=-601).timestamp,
+            'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
+            'lastTradeTimestamp': None,
+            'symbol': 'LTC/USDT',
+            'type': 'limit',
+            'side': 'buy',
+            'price': 0.016804,
+            'amount': 0.55,
+            'cost': 0.0,
+            'average': None,
+            'filled': 0.0,
+            'remaining': 0.55,
+            'status': 'canceled',
+            'fee': None,
+            'trades': None
+        }
+
+
 @pytest.fixture
 def limit_sell_order():
     return {
@@ -1329,6 +1423,15 @@ def trades_for_order():
 
 @pytest.fixture(scope="function")
 def trades_history():
+    return [[1565798399463, '126181329', None, 'buy', 0.019627, 0.04, 0.00078508],
+            [1565798399629, '126181330', None, 'buy', 0.019627, 0.244, 0.004788987999999999],
+            [1565798399752, '126181331', None, 'sell', 0.019626, 0.011, 0.00021588599999999999],
+            [1565798399862, '126181332', None, 'sell', 0.019626, 0.011, 0.00021588599999999999],
+            [1565798399872, '126181333', None, 'sell', 0.019626, 0.011, 0.00021588599999999999]]
+
+
+@pytest.fixture(scope="function")
+def fetch_trades_result():
     return [{'info': {'a': 126181329,
                       'p': '0.01962700',
                       'q': '0.04000000',
@@ -1483,7 +1586,7 @@ def buy_order_fee():
         'id': 'mocked_limit_buy_old',
         'type': 'limit',
         'side': 'buy',
-        'pair': 'mocked',
+        'symbol': 'mocked',
         'datetime': str(arrow.utcnow().shift(minutes=-601).datetime),
         'price': 0.245441,
         'amount': 8.0,

tests/data/test_converter.py

@@ -5,12 +5,10 @@ from freqtrade.configuration.timerange import TimeRange
 from freqtrade.data.converter import (convert_ohlcv_format,
                                       convert_trades_format,
                                       ohlcv_fill_up_missing_data,
-                                      ohlcv_to_dataframe,
-                                      trim_dataframe)
-from freqtrade.data.history import (get_timerange,
-                                    load_data,
-                                    load_pair_history,
-                                    validate_backtest_data)
+                                      ohlcv_to_dataframe, trades_dict_to_list,
+                                      trades_remove_duplicates, trim_dataframe)
+from freqtrade.data.history import (get_timerange, load_data,
+                                    load_pair_history, validate_backtest_data)
 from tests.conftest import log_has
 from tests.data.test_history import _backup_file, _clean_test_file
 
@@ -197,32 +195,60 @@ def test_trim_dataframe(testdatadir) -> None:
     assert all(data_modify.iloc[0] == data.iloc[25])
 
 
-def test_convert_trades_format(mocker, default_conf, testdatadir):
-    file = testdatadir / "XRP_ETH-trades.json.gz"
-    file_new = testdatadir / "XRP_ETH-trades.json"
-    _backup_file(file, copy_file=True)
-    default_conf['datadir'] = testdatadir
+def test_trades_remove_duplicates(trades_history):
+    trades_history1 = trades_history * 3
+    assert len(trades_history1) == len(trades_history) * 3
+    res = trades_remove_duplicates(trades_history1)
+    assert len(res) == len(trades_history)
+    for i, t in enumerate(res):
+        assert t == trades_history[i]
 
-    assert not file_new.exists()
+
+def test_trades_dict_to_list(fetch_trades_result):
+    res = trades_dict_to_list(fetch_trades_result)
+    assert isinstance(res, list)
+    assert isinstance(res[0], list)
+    for i, t in enumerate(res):
+        assert t[0] == fetch_trades_result[i]['timestamp']
+        assert t[1] == fetch_trades_result[i]['id']
+        assert t[2] == fetch_trades_result[i]['type']
+        assert t[3] == fetch_trades_result[i]['side']
+        assert t[4] == fetch_trades_result[i]['price']
+        assert t[5] == fetch_trades_result[i]['amount']
+        assert t[6] == fetch_trades_result[i]['cost']
+
+
+def test_convert_trades_format(mocker, default_conf, testdatadir):
+    files = [{'old': testdatadir / "XRP_ETH-trades.json.gz",
+              'new': testdatadir / "XRP_ETH-trades.json"},
+             {'old': testdatadir / "XRP_OLD-trades.json.gz",
+              'new': testdatadir / "XRP_OLD-trades.json"},
+             ]
+    for file in files:
+        _backup_file(file['old'], copy_file=True)
+        assert not file['new'].exists()
+
+    default_conf['datadir'] = testdatadir
 
     convert_trades_format(default_conf, convert_from='jsongz',
                           convert_to='json', erase=False)
 
-    assert file_new.exists()
-    assert file.exists()
+    for file in files:
+        assert file['new'].exists()
+        assert file['old'].exists()
 
         # Remove original file
-    file.unlink()
+        file['old'].unlink()
     # Convert back
     convert_trades_format(default_conf, convert_from='json',
                           convert_to='jsongz', erase=True)
+    for file in files:
+        assert file['old'].exists()
+        assert not file['new'].exists()
 
-    assert file.exists()
-    assert not file_new.exists()
-
-    _clean_test_file(file)
-    if file_new.exists():
-        file_new.unlink()
+        _clean_test_file(file['old'])
+        if file['new'].exists():
+            file['new'].unlink()
 
 
 def test_convert_ohlcv_format(mocker, default_conf, testdatadir):

tests/data/test_dataprovider.py

@@ -1,8 +1,11 @@
 from unittest.mock import MagicMock
 
 from pandas import DataFrame
+import pytest
 
 from freqtrade.data.dataprovider import DataProvider
+from freqtrade.pairlist.pairlistmanager import PairListManager
+from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.state import RunMode
 from tests.conftest import get_patched_exchange
 
@@ -64,8 +67,8 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history):
     assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty
 
     # Test with and without parameter
-    assert dp.get_pair_dataframe("UNITTEST/BTC",
-                                 ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC"))
+    assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)\
+        .equals(dp.get_pair_dataframe("UNITTEST/BTC"))
 
     default_conf["runmode"] = RunMode.LIVE
     dp = DataProvider(default_conf, exchange)
@@ -90,10 +93,7 @@ def test_available_pairs(mocker, default_conf, ohlcv_history):
 
     dp = DataProvider(default_conf, exchange)
     assert len(dp.available_pairs) == 2
-    assert dp.available_pairs == [
-        ("XRP/BTC", ticker_interval),
-        ("UNITTEST/BTC", ticker_interval),
-    ]
+    assert dp.available_pairs == [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval), ]
 
 
 def test_refresh(mocker, default_conf, ohlcv_history):
@@ -152,3 +152,45 @@ def test_market(mocker, default_conf, markets):
 
     res = dp.market('UNITTEST/BTC')
     assert res is None
+
+
+def test_ticker(mocker, default_conf, tickers):
+    ticker_mock = MagicMock(return_value=tickers()['ETH/BTC'])
+    mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock)
+    exchange = get_patched_exchange(mocker, default_conf)
+    dp = DataProvider(default_conf, exchange)
+    res = dp.ticker('ETH/BTC')
+    assert type(res) is dict
+    assert 'symbol' in res
+    assert res['symbol'] == 'ETH/BTC'
+
+    ticker_mock = MagicMock(side_effect=DependencyException('Pair not found'))
+    mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock)
+    exchange = get_patched_exchange(mocker, default_conf)
+    dp = DataProvider(default_conf, exchange)
+    res = dp.ticker('UNITTEST/BTC')
+    assert res == {}
+
+
+def test_current_whitelist(mocker, default_conf, tickers):
+    # patch default conf to volumepairlist
+    default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5}
+
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers)
+    exchange = get_patched_exchange(mocker, default_conf)
+
+    pairlist = PairListManager(exchange, default_conf)
+    dp = DataProvider(default_conf, exchange, pairlist)
+
+    # Simulate volumepairs from exchange.
+    pairlist.refresh_pairlist()
+
+    assert dp.current_whitelist() == pairlist._whitelist
+    # The identity of the 2 lists should be identical
+    assert dp.current_whitelist() is pairlist._whitelist
+
+    with pytest.raises(OperationalException):
+        dp = DataProvider(default_conf, exchange)
+        dp.current_whitelist()

tests/data/test_history.py

@@ -547,6 +547,17 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     assert log_has("New Amount of trades: 5", caplog)
     assert file1.is_file()
 
+    ght_mock.reset_mock()
+    since_time = int(trades_history[-3][0] // 1000)
+    since_time2 = int(trades_history[-1][0] // 1000)
+    timerange = TimeRange('date', None, since_time, 0)
+    assert _download_trades_history(data_handler=data_handler, exchange=exchange,
+                                    pair='ETH/BTC', timerange=timerange)
+
+    assert ght_mock.call_count == 1
+    # Check this in seconds - since we had to convert to seconds above too.
+    assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time2 - 5
+
     # clean files freshly downloaded
     _clean_test_file(file1)
 
@@ -601,7 +612,7 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir):
 def test_jsondatahandler_trades_get_pairs(testdatadir):
     pairs = JsonGzDataHandler.trades_get_pairs(testdatadir)
     # Convert to set to avoid failures due to sorting
-    assert set(pairs) == {'XRP/ETH'}
+    assert set(pairs) == {'XRP/ETH', 'XRP/OLD'}
 
 
 def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
@@ -614,6 +625,17 @@ def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
     assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m')
 
 
+def test_jsondatahandler_trades_load(mocker, testdatadir, caplog):
+    dh = JsonGzDataHandler(testdatadir)
+    logmsg = "Old trades format detected - converting"
+    dh.trades_load('XRP/ETH')
+    assert not log_has(logmsg, caplog)
+
+    # Test conversation is happening
+    dh.trades_load('XRP/OLD')
+    assert log_has(logmsg, caplog)
+
+
 def test_jsondatahandler_trades_purge(mocker, testdatadir):
     mocker.patch.object(Path, "exists", MagicMock(return_value=False))
     mocker.patch.object(Path, "unlink", MagicMock())

tests/edge/test_edge.py

@@ -335,12 +335,16 @@ def test_edge_init_error(mocker, edge_conf,):
         get_patched_freqtradebot(mocker, edge_conf)
 
 
-def test_process_expectancy(mocker, edge_conf):
+@pytest.mark.parametrize("fee,risk_reward_ratio,expectancy", [
+    (0.0005, 306.5384615384, 101.5128205128),
+    (0.001, 152.6923076923, 50.2307692308),
+])
+def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectancy):
     edge_conf['edge']['min_trade_number'] = 2
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
 
     def get_fee(*args, **kwargs):
-        return 0.001
+        return fee
 
     freqtrade.exchange.get_fee = get_fee
     edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
@@ -394,9 +398,9 @@ def test_process_expectancy(mocker, edge_conf):
     assert 'TEST/BTC' in final
     assert final['TEST/BTC'].stoploss == -0.9
     assert round(final['TEST/BTC'].winrate, 10) == 0.3333333333
-    assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384
+    assert round(final['TEST/BTC'].risk_reward_ratio, 10) == risk_reward_ratio
     assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0
-    assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128
+    assert round(final['TEST/BTC'].expectancy, 10) == expectancy
 
     # Pop last item so no trade is profitable
     trades.pop()

tests/exchange/test_binance.py

@@ -9,7 +9,12 @@ from freqtrade.exceptions import (DependencyException, InvalidOrderException,
 from tests.conftest import get_patched_exchange
 
 
-def test_stoploss_order_binance(default_conf, mocker):
+@pytest.mark.parametrize('limitratio,expected', [
+    (None, 220 * 0.99),
+    (0.99, 220 * 0.99),
+    (0.98, 220 * 0.98),
+])
+def test_stoploss_order_binance(default_conf, mocker, limitratio, expected):
     api_mock = MagicMock()
     order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6))
     order_type = 'stop_loss_limit'
@@ -20,7 +25,6 @@ def test_stoploss_order_binance(default_conf, mocker):
             'foo': 'bar'
         }
     })
-
     default_conf['dry_run'] = False
     mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y)
     mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
@@ -32,8 +36,8 @@ def test_stoploss_order_binance(default_conf, mocker):
                                   order_types={'stoploss_on_exchange_limit_ratio': 1.05})
 
     api_mock.create_order.reset_mock()
-
-    order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={})
+    order_types = {} if limitratio is None else {'stoploss_on_exchange_limit_ratio': limitratio}
+    order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types=order_types)
 
     assert 'id' in order
     assert 'info' in order
@@ -42,7 +46,8 @@ def test_stoploss_order_binance(default_conf, mocker):
     assert api_mock.create_order.call_args_list[0][1]['type'] == order_type
     assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell'
     assert api_mock.create_order.call_args_list[0][1]['amount'] == 1
-    assert api_mock.create_order.call_args_list[0][1]['price'] == 220
+    # Price should be 1% below stopprice
+    assert api_mock.create_order.call_args_list[0][1]['price'] == expected
     assert api_mock.create_order.call_args_list[0][1]['params'] == {'stopPrice': 220}
 
     # test exception handling

tests/exchange/test_exchange.py

@@ -1537,18 +1537,18 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na
 @pytest.mark.asyncio
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name,
-                                   trades_history):
+                                   fetch_trades_result):
 
     caplog.set_level(logging.DEBUG)
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
     # Monkey-patch async function
-    exchange._api_async.fetch_trades = get_mock_coro(trades_history)
+    exchange._api_async.fetch_trades = get_mock_coro(fetch_trades_result)
 
     pair = 'ETH/BTC'
     res = await exchange._async_fetch_trades(pair, since=None, params=None)
     assert type(res) is list
-    assert isinstance(res[0], dict)
-    assert isinstance(res[1], dict)
+    assert isinstance(res[0], list)
+    assert isinstance(res[1], list)
 
     assert exchange._api_async.fetch_trades.call_count == 1
     assert exchange._api_async.fetch_trades.call_args[0][0] == pair
@@ -1594,7 +1594,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang
         if 'since' in kwargs:
             # Return first 3
             return trades_history[:-2]
-        elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3]['id']:
+        elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3][1]:
             # Return 2
             return trades_history[-3:-1]
         else:
@@ -1604,8 +1604,8 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang
     exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
 
     pair = 'ETH/BTC'
-    ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0]["timestamp"],
-                                                     until=trades_history[-1]["timestamp"]-1)
+    ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0][0],
+                                                     until=trades_history[-1][0]-1)
     assert type(ret) is tuple
     assert ret[0] == pair
     assert type(ret[1]) is list
@@ -1614,7 +1614,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang
     fetch_trades_cal = exchange._async_fetch_trades.call_args_list
     # first call (using since, not fromId)
     assert fetch_trades_cal[0][0][0] == pair
-    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]
 
     # 2nd call
     assert fetch_trades_cal[1][0][0] == pair
@@ -1630,7 +1630,7 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
     caplog.set_level(logging.DEBUG)
 
     async def mock_get_trade_hist(pair, *args, **kwargs):
-        if kwargs['since'] == trades_history[0]["timestamp"]:
+        if kwargs['since'] == trades_history[0][0]:
             return trades_history[:-1]
         else:
             return trades_history[-1:]
@@ -1640,8 +1640,8 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
     # Monkey-patch async function
     exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
     pair = 'ETH/BTC'
-    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
-                                                       until=trades_history[-1]["timestamp"]-1)
+    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0],
+                                                       until=trades_history[-1][0]-1)
     assert type(ret) is tuple
     assert ret[0] == pair
     assert type(ret[1]) is list
@@ -1650,11 +1650,11 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha
     fetch_trades_cal = exchange._async_fetch_trades.call_args_list
     # first call (using since, not fromId)
     assert fetch_trades_cal[0][0][0] == pair
-    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]
 
     # 2nd call
     assert fetch_trades_cal[1][0][0] == pair
-    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]
     assert log_has_re(r"Stopping because until was reached.*", caplog)
 
 
@@ -1666,7 +1666,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
     caplog.set_level(logging.DEBUG)
 
     async def mock_get_trade_hist(pair, *args, **kwargs):
-        if kwargs['since'] == trades_history[0]["timestamp"]:
+        if kwargs['since'] == trades_history[0][0]:
             return trades_history[:-1]
         else:
             return []
@@ -1676,8 +1676,8 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
     # Monkey-patch async function
     exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
     pair = 'ETH/BTC'
-    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"],
-                                                       until=trades_history[-1]["timestamp"]-1)
+    ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0],
+                                                       until=trades_history[-1][0]-1)
     assert type(ret) is tuple
     assert ret[0] == pair
     assert type(ret[1]) is list
@@ -1686,7 +1686,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog,
     fetch_trades_cal = exchange._async_fetch_trades.call_args_list
     # first call (using since, not fromId)
     assert fetch_trades_cal[0][0][0] == pair
-    assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"]
+    assert fetch_trades_cal[0][1]['since'] == trades_history[0][0]
 
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
@@ -1698,8 +1698,8 @@ def test_get_historic_trades(default_conf, mocker, caplog, exchange_name, trades
 
     exchange._async_get_trade_history_id = get_mock_coro((pair, trades_history))
     exchange._async_get_trade_history_time = get_mock_coro((pair, trades_history))
-    ret = exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
-                                       until=trades_history[-1]["timestamp"])
+    ret = exchange.get_historic_trades(pair, since=trades_history[0][0],
+                                       until=trades_history[-1][0])
 
     # Depending on the exchange, one or the other method should be called
     assert sum([exchange._async_get_trade_history_id.call_count,
@@ -1720,8 +1720,8 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange
 
     with pytest.raises(OperationalException,
                        match="This exchange does not suport downloading Trades."):
-        exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"],
-                                     until=trades_history[-1]["timestamp"])
+        exchange.get_historic_trades(pair, since=trades_history[0][0],
+                                     until=trades_history[-1][0])
 
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
@@ -2145,3 +2145,58 @@ def test_symbol_is_pair(market_symbol, base_currency, quote_currency, expected_r
 ])
 def test_market_is_active(market, expected_result) -> None:
     assert market_is_active(market) == expected_result
+
+
+@pytest.mark.parametrize("order,expected", [
+    ([{'fee'}], False),
+    ({'fee': None}, False),
+    ({'fee': {'currency': 'ETH/BTC'}}, False),
+    ({'fee': {'currency': 'ETH/BTC', 'cost': None}}, False),
+    ({'fee': {'currency': 'ETH/BTC', 'cost': 0.01}}, True),
+])
+def test_order_has_fee(order, expected) -> None:
+    assert Exchange.order_has_fee(order) == expected
+
+
+@pytest.mark.parametrize("order,expected", [
+    ({'symbol': 'ETH/BTC', 'fee': {'currency': 'ETH', 'cost': 0.43}},
+        (0.43, 'ETH', 0.01)),
+    ({'symbol': 'ETH/USDT', 'fee': {'currency': 'USDT', 'cost': 0.01}},
+        (0.01, 'USDT', 0.01)),
+    ({'symbol': 'BTC/USDT', 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}},
+        (0.34, 'USDT', 0.01)),
+])
+def test_extract_cost_curr_rate(mocker, default_conf, order, expected) -> None:
+    mocker.patch('freqtrade.exchange.Exchange.calculate_fee_rate', MagicMock(return_value=0.01))
+    ex = get_patched_exchange(mocker, default_conf)
+    assert ex.extract_cost_curr_rate(order) == expected
+
+
+@pytest.mark.parametrize("order,expected", [
+    # Using base-currency
+    ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05,
+        'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.1),
+    ({'symbol': 'ETH/BTC', 'amount': 0.05, 'cost': 0.05,
+        'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.08),
+    # Using quote currency
+    ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05,
+        'fee': {'currency': 'BTC', 'cost': 0.005}}, 0.1),
+    ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05,
+        'fee': {'currency': 'BTC', 'cost': 0.002, 'rate': None}}, 0.04),
+    # Using foreign currency
+    ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05,
+        'fee': {'currency': 'NEO', 'cost': 0.0012}}, 0.001944),
+    ({'symbol': 'ETH/BTC', 'amount': 2.21, 'cost': 0.02992561,
+        'fee': {'currency': 'NEO', 'cost': 0.00027452}}, 0.00074305),
+    # TODO: More tests here!
+    # Rate included in return - return as is
+    ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05,
+        'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}}, 0.01),
+    ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05,
+        'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.005}}, 0.005),
+])
+def test_calculate_fee_rate(mocker, default_conf, order, expected) -> None:
+    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'last': 0.081})
+
+    ex = get_patched_exchange(mocker, default_conf)
+    assert ex.calculate_fee_rate(order) == expected

tests/optimize/test_backtesting.py

@@ -649,6 +649,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
         assert log_has(line, caplog)
 
 
+@pytest.mark.filterwarnings("ignore:deprecated")
 def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
 
     patch_exchange(mocker)

tests/rpc/test_rpc.py

@@ -51,7 +51,11 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'open_date_hum': ANY,
         'is_open': ANY,
         'fee_open': ANY,
+        'fee_open_cost': ANY,
+        'fee_open_currency': ANY,
         'fee_close': ANY,
+        'fee_close_cost': ANY,
+        'fee_close_currency': ANY,
         'open_rate_requested': ANY,
         'open_trade_price': ANY,
         'close_rate_requested': ANY,
@@ -90,7 +94,11 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'open_date_hum': ANY,
         'is_open': ANY,
         'fee_open': ANY,
+        'fee_open_cost': ANY,
+        'fee_open_currency': ANY,
         'fee_close': ANY,
+        'fee_close_cost': ANY,
+        'fee_close_currency': ANY,
         'open_rate_requested': ANY,
         'open_trade_price': ANY,
         'close_rate_requested': ANY,

tests/rpc/test_rpc_apiserver.py

@@ -49,6 +49,7 @@ def client_get(client, url):
 def assert_response(response, expected_code=200):
     assert response.status_code == expected_code
     assert response.content_type == "application/json"
+    assert ('Access-Control-Allow-Origin', '*') in response.headers._list
 
 
 def test_api_not_found(botclient):
@@ -94,6 +95,33 @@ def test_api_unauthorized(botclient):
     assert rc.json == {'error': 'Unauthorized'}
 
 
+def test_api_token_login(botclient):
+    ftbot, client = botclient
+    rc = client_post(client, f"{BASE_URI}/token/login")
+    assert_response(rc)
+    assert 'access_token' in rc.json
+    assert 'refresh_token' in rc.json
+
+    # test Authentication is working with JWT tokens too
+    rc = client.get(f"{BASE_URI}/count",
+                    content_type="application/json",
+                    headers={'Authorization': f'Bearer {rc.json["access_token"]}'})
+    assert_response(rc)
+
+
+def test_api_token_refresh(botclient):
+    ftbot, client = botclient
+    rc = client_post(client, f"{BASE_URI}/token/login")
+    assert_response(rc)
+    rc = client.post(f"{BASE_URI}/token/refresh",
+                     content_type="application/json",
+                     data=None,
+                     headers={'Authorization': f'Bearer {rc.json["refresh_token"]}'})
+    assert_response(rc)
+    assert 'access_token' in rc.json
+    assert 'refresh_token' not in rc.json
+
+
 def test_api_stop_workflow(botclient):
     ftbot, client = botclient
     assert ftbot.state == State.RUNNING
@@ -123,6 +151,12 @@ def test_api__init__(default_conf, mocker):
     """
     Test __init__() method
     """
+    default_conf.update({"api_server": {"enabled": True,
+                                        "listen_ip_address": "127.0.0.1",
+                                        "listen_port": 8080,
+                                        "username": "TestUser",
+                                        "password": "testPass",
+                                        }})
     mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock())
     mocker.patch('freqtrade.rpc.api_server.ApiServer.run', MagicMock())
 
@@ -283,6 +317,7 @@ def test_api_show_config(botclient, mocker):
     assert 'dry_run' in rc.json
     assert rc.json['exchange'] == 'bittrex'
     assert rc.json['ticker_interval'] == '5m'
+    assert rc.json['state'] == 'running'
     assert not rc.json['trailing_stop']
 
 
@@ -472,7 +507,11 @@ def test_api_status(botclient, mocker, ticker, fee, markets):
                         'close_rate_requested': None,
                         'current_rate': 1.099e-05,
                         'fee_close': 0.0025,
+                        'fee_close_cost': None,
+                        'fee_close_currency': None,
                         'fee_open': 0.0025,
+                        'fee_open_cost': None,
+                        'fee_open_currency': None,
                         'open_date': ANY,
                         'is_open': True,
                         'max_rate': 0.0,
@@ -575,7 +614,11 @@ def test_api_forcebuy(botclient, mocker, fee):
                        'close_profit': None,
                        'close_rate_requested': None,
                        'fee_close': 0.0025,
+                       'fee_close_cost': None,
+                       'fee_close_currency': None,
                        'fee_open': 0.0025,
+                       'fee_open_cost': None,
+                       'fee_open_currency': None,
                        'is_open': False,
                        'max_rate': None,
                        'min_rate': None,

tests/test_freqtradebot.py

@@ -1158,7 +1158,8 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
         'status': 'closed',
         'type': 'stop_loss_limit',
         'price': 3,
-        'average': 2
+        'average': 2,
+        'amount': limit_buy_order['amount'],
     })
     mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit)
     assert freqtrade.handle_stoploss_on_exchange(trade) is True
@@ -2220,6 +2221,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
                                            limit_buy_order_old_partial_canceled, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
     cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled)
+    mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=0))
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
@@ -2239,7 +2241,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
     # and apply fees if necessary.
     freqtrade.check_handle_timedout()
 
-    assert log_has_re(r"Applying fee on amount for Trade.* Order", caplog)
+    assert log_has_re(r"Applying fee on amount for Trade.*", caplog)
 
     assert cancel_order_mock.call_count == 1
     assert rpc_mock.call_count == 2
@@ -2247,9 +2249,10 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
     assert len(trades) == 1
     # Verify that trade has been updated
     assert trades[0].amount == (limit_buy_order_old_partial['amount'] -
-                                limit_buy_order_old_partial['remaining']) - 0.0001
+                                limit_buy_order_old_partial['remaining']) - 0.023
     assert trades[0].open_order_id is None
-    assert trades[0].fee_open == 0
+    assert trades[0].fee_updated('buy')
+    assert pytest.approx(trades[0].fee_open) == 0.001
 
 
 def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee,
@@ -2320,7 +2323,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke
                       caplog)
 
 
-def test_handle_timedout_limit_buy(mocker, caplog, default_conf, limit_buy_order) -> None:
+def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     cancel_order_mock = MagicMock(return_value=limit_buy_order)
@@ -2332,27 +2335,51 @@ def test_handle_timedout_limit_buy(mocker, caplog, default_conf, limit_buy_order
     Trade.session = MagicMock()
     trade = MagicMock()
     trade.pair = 'LTC/ETH'
-    limit_buy_order['remaining'] = limit_buy_order['amount']
+    limit_buy_order['filled'] = 0.0
+    limit_buy_order['status'] = 'open'
     reason = CANCEL_REASON['TIMEOUT']
     assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
     assert cancel_order_mock.call_count == 1
 
     cancel_order_mock.reset_mock()
-    limit_buy_order['amount'] = 2
+    limit_buy_order['filled'] = 2
     assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
     assert cancel_order_mock.call_count == 1
 
+    limit_buy_order['filled'] = 2
     mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException)
     assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
 
 
+@pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'],
+                         indirect=['limit_buy_order_canceled_empty'])
+def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf,
+                                     limit_buy_order_canceled_empty) -> None:
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    cancel_order_mock = mocker.patch(
+        'freqtrade.exchange.Exchange.cancel_order_with_result',
+        return_value=limit_buy_order_canceled_empty)
+    nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_buy_cancel')
+    freqtrade = FreqtradeBot(default_conf)
+
+    Trade.session = MagicMock()
+    reason = CANCEL_REASON['TIMEOUT']
+    trade = MagicMock()
+    trade.pair = 'LTC/ETH'
+    assert freqtrade.handle_cancel_buy(trade, limit_buy_order_canceled_empty, reason)
+    assert cancel_order_mock.call_count == 0
+    assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog)
+    assert nofiy_mock.call_count == 1
+
+
 @pytest.mark.parametrize('cancelorder', [
     {},
     {'remaining': None},
     'String Return value',
     123
 ])
-def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_order,
+def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order,
                                         cancelorder) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -2368,18 +2395,19 @@ def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_
     Trade.session = MagicMock()
     trade = MagicMock()
     trade.pair = 'LTC/ETH'
-    limit_buy_order['remaining'] = limit_buy_order['amount']
+    limit_buy_order['filled'] = 0.0
+    limit_buy_order['status'] = 'open'
     reason = CANCEL_REASON['TIMEOUT']
     assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
     assert cancel_order_mock.call_count == 1
 
     cancel_order_mock.reset_mock()
-    limit_buy_order['amount'] = 2
+    limit_buy_order['filled'] = 1.0
     assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
     assert cancel_order_mock.call_count == 1
 
 
-def test_handle_timedout_limit_sell(mocker, default_conf) -> None:
+def test_handle_cancel_sell_limit(mocker, default_conf) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     cancel_order_mock = MagicMock()
@@ -2405,6 +2433,22 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None:
     assert cancel_order_mock.call_count == 1
 
 
+def test_handle_cancel_sell_cancel_exception(mocker, default_conf) -> None:
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException())
+
+    freqtrade = FreqtradeBot(default_conf)
+
+    trade = MagicMock()
+    reason = CANCEL_REASON['TIMEOUT']
+    order = {'remaining': 1,
+             'amount': 1,
+             'status': "open"}
+    assert freqtrade.handle_cancel_sell(trade, order, reason) == 'error cancelling order'
+
+
 def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -3309,8 +3353,6 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order,
 
 def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker):
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     amount = sum(x['amount'] for x in trades_for_order)
     trade = Trade(
         pair='LTC/ETH',
@@ -3321,21 +3363,43 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe
         fee_close=fee.return_value,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount is reduced by "fee"
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001)
     assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
-                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades',
+                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).',
                    caplog)
 
 
+def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee,
+                                    caplog, mocker):
+    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
+    walletmock = mocker.patch('freqtrade.wallets.Wallets.update')
+    mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=8.1122)
+    amount = sum(x['amount'] for x in trades_for_order)
+    trade = Trade(
+        pair='LTC/ETH',
+        amount=amount,
+        exchange='binance',
+        open_rate=0.245441,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_order_id="123456"
+    )
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+
+    walletmock.reset_mock()
+    # Amount is kept as is
+    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
+    assert walletmock.call_count == 1
+    assert log_has_re(r'Fee amount for Trade.* was in base currency '
+                      '- Eating Fee 0.008 into dust', caplog)
+
+
 def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee):
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[])
 
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     amount = buy_order_fee['amount']
     trade = Trade(
         pair='LTC/ETH',
@@ -3346,8 +3410,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f
         fee_close=fee.return_value,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount is reduced by "fee"
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
@@ -3359,8 +3422,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f
 def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker):
     trades_for_order[0]['fee']['currency'] = 'ETH'
 
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
     amount = sum(x['amount'] for x in trades_for_order)
     trade = Trade(
@@ -3372,8 +3433,7 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fe
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount does not change
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
@@ -3386,8 +3446,6 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_
     limit_buy_order['fee'] = {'cost': 0.004, 'currency': None}
     trades_for_order[0]['fee']['currency'] = None
 
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
     amount = sum(x['amount'] for x in trades_for_order)
     trade = Trade(
@@ -3399,8 +3457,7 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount does not change
     assert freqtrade.get_real_amount(trade, limit_buy_order) == amount
@@ -3410,8 +3467,6 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee,
     trades_for_order[0]['fee']['currency'] = 'BNB'
     trades_for_order[0]['fee']['cost'] = 0.00094518
 
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
     amount = sum(x['amount'] for x in trades_for_order)
     trade = Trade(
@@ -3423,16 +3478,13 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee,
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount does not change
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
 
 
 def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker):
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2)
     amount = float(sum(x['amount'] for x in trades_for_order2))
     trade = Trade(
@@ -3444,13 +3496,12 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount is reduced by "fee"
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001)
     assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
-                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades',
+                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).',
                    caplog)
 
 
@@ -3459,8 +3510,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee
     limit_buy_order = deepcopy(buy_order_fee)
     limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'}
 
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order',
                  return_value=[trades_for_order])
     amount = float(sum(x['amount'] for x in trades_for_order))
@@ -3473,13 +3522,12 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount is reduced by "fee"
     assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004
     assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
-                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996) from Order',
+                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).',
                    caplog)
 
 
@@ -3487,8 +3535,6 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order
     limit_buy_order = deepcopy(buy_order_fee)
     limit_buy_order['fee'] = {'cost': 0.004}
 
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[])
     amount = float(sum(x['amount'] for x in trades_for_order))
     trade = Trade(
@@ -3500,8 +3546,7 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount does not change
     assert freqtrade.get_real_amount(trade, limit_buy_order) == amount
@@ -3511,8 +3556,6 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_
     limit_buy_order = deepcopy(buy_order_fee)
     limit_buy_order['amount'] = limit_buy_order['amount'] - 0.001
 
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
     amount = float(sum(x['amount'] for x in trades_for_order))
     trade = Trade(
@@ -3524,8 +3567,7 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_
         fee_close=fee.return_value,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount does not change
     with pytest.raises(DependencyException, match=r"Half bought\? Amounts don't match"):
@@ -3538,8 +3580,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b
     limit_buy_order = deepcopy(buy_order_fee)
     trades_for_order[0]['amount'] = trades_for_order[0]['amount'] + 1e-15
 
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
     amount = float(sum(x['amount'] for x in trades_for_order))
     trade = Trade(
@@ -3551,8 +3591,7 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     # Amount changes by fee amount.
     assert isclose(freqtrade.get_real_amount(trade, limit_buy_order), amount - (amount * 0.001),
@@ -3563,8 +3602,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee,
     # Remove "Currency" from fee dict
     trades_for_order[0]['fee'] = {'cost': 0.008}
 
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
     amount = sum(x['amount'] for x in trades_for_order)
     trade = Trade(
@@ -3577,15 +3614,12 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee,
 
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
     # Amount does not change
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
 
 
 def test_get_real_amount_open_trade(default_conf, fee, mocker):
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
     amount = 12345
     trade = Trade(
         pair='LTC/ETH',
@@ -3600,12 +3634,41 @@ def test_get_real_amount_open_trade(default_conf, fee, mocker):
         'id': 'mocked_order',
         'amount': amount,
         'status': 'open',
+        'side': 'buy',
     }
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
     assert freqtrade.get_real_amount(trade, order) == amount
 
 
+@pytest.mark.parametrize('amount,fee_abs,wallet,amount_exp', [
+    (8.0, 0.0, 10, 8),
+    (8.0, 0.0, 0, 8),
+    (8.0, 0.1, 0, 7.9),
+    (8.0, 0.1, 10, 8),
+    (8.0, 0.1, 8.0, 8.0),
+    (8.0, 0.1, 7.9, 7.9),
+])
+def test_apply_fee_conditional(default_conf, fee, caplog, mocker,
+                               amount, fee_abs, wallet, amount_exp):
+    walletmock = mocker.patch('freqtrade.wallets.Wallets.update')
+    mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=wallet)
+    trade = Trade(
+        pair='LTC/ETH',
+        amount=amount,
+        exchange='binance',
+        open_rate=0.245441,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_order_id="123456"
+    )
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+
+    walletmock.reset_mock()
+    # Amount is kept as is
+    assert freqtrade.apply_fee_conditional(trade, 'LTC', amount, fee_abs) == amount_exp
+    assert walletmock.call_count == 1
+
+
 def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker,
                                     order_book_l2):
     default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True
@@ -3899,4 +3962,3 @@ def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limi
     freqtrade.cancel_all_open_orders()
     assert buy_mock.call_count == 1
     assert sell_mock.call_count == 1
-

tests/test_integration.py

@@ -44,6 +44,8 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
     }
     stoploss_order_closed = stoploss_order_open.copy()
     stoploss_order_closed['status'] = 'closed'
+    stoploss_order_closed['filled'] = stoploss_order_closed['amount']
+
     # Sell first trade based on stoploss, keep 2nd and 3rd trade open
     stoploss_order_mock = MagicMock(
         side_effect=[stoploss_order_closed, stoploss_order_open, stoploss_order_open])
@@ -67,7 +69,6 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
     mocker.patch.multiple(
         'freqtrade.freqtradebot.FreqtradeBot',
         create_stoploss_order=MagicMock(return_value=True),
-        update_trade_state=MagicMock(),
         _notify_sell=MagicMock(),
     )
     mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock)
@@ -97,8 +98,9 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
 
     # Only order for 3rd trade needs to be cancelled
     assert cancel_order_mock.call_count == 1
-    # Wallets must be updated between stoploss cancellation and selling.
-    assert wallets_mock.call_count == 2
+    # Wallets must be updated between stoploss cancellation and selling, and will be updated again
+    # during update_trade_state
+    assert wallets_mock.call_count == 4
 
     trade = trades[0]
     assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value
@@ -144,7 +146,6 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc
     mocker.patch.multiple(
         'freqtrade.freqtradebot.FreqtradeBot',
         create_stoploss_order=MagicMock(return_value=True),
-        update_trade_state=MagicMock(),
         _notify_sell=MagicMock(),
     )
     should_sell_mock = MagicMock(side_effect=[

tests/test_main.py

@@ -115,6 +115,32 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None:
     assert log_has('Oh snap!', caplog)
 
 
+def test_main_operational_exception1(mocker, default_conf, caplog) -> None:
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.commands.list_commands.available_exchanges',
+        MagicMock(side_effect=ValueError('Oh snap!'))
+    )
+    patched_configuration_load_config_file(mocker, default_conf)
+
+    args = ['list-exchanges']
+
+    # Test Main + the KeyboardInterrupt exception
+    with pytest.raises(SystemExit):
+        main(args)
+
+    assert log_has('Fatal exception!', caplog)
+    assert not log_has_re(r'SIGINT.*', caplog)
+    mocker.patch(
+        'freqtrade.commands.list_commands.available_exchanges',
+        MagicMock(side_effect=KeyboardInterrupt)
+    )
+    with pytest.raises(SystemExit):
+        main(args)
+
+    assert log_has_re(r'SIGINT.*', caplog)
+
+
 def test_main_reload_conf(mocker, default_conf, caplog) -> None:
     patch_exchange(mocker)
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cleanup', MagicMock())

tests/test_persistence.py

@@ -465,6 +465,10 @@ def test_migrate_old(mocker, default_conf, fee):
     assert trade.initial_stop_loss == 0.0
     assert trade.open_trade_price == trade._calc_open_trade_price()
     assert trade.close_profit_abs is None
+    assert trade.fee_open_cost is None
+    assert trade.fee_open_currency is None
+    assert trade.fee_close_cost is None
+    assert trade.fee_close_currency is None
 
     trade = Trade.query.filter(Trade.id == 2).first()
     assert trade.close_rate is not None
@@ -741,7 +745,11 @@ def test_to_json(default_conf, fee):
                       'open_rate_requested': None,
                       'open_trade_price': 15.1668225,
                       'fee_close': 0.0025,
+                      'fee_close_cost': None,
+                      'fee_close_currency': None,
                       'fee_open': 0.0025,
+                      'fee_open_cost': None,
+                      'fee_open_currency': None,
                       'close_rate': None,
                       'close_rate_requested': None,
                       'amount': 123.0,
@@ -790,7 +798,11 @@ def test_to_json(default_conf, fee):
                       'close_profit': None,
                       'close_rate_requested': None,
                       'fee_close': 0.0025,
+                      'fee_close_cost': None,
+                      'fee_close_currency': None,
                       'fee_open': 0.0025,
+                      'fee_open_cost': None,
+                      'fee_open_currency': None,
                       'is_open': None,
                       'max_rate': None,
                       'min_rate': None,
@@ -862,6 +874,75 @@ def test_stoploss_reinitialization(default_conf, fee):
     assert trade_adj.initial_stop_loss_pct == -0.04
 
 
+def test_update_fee(fee):
+    trade = Trade(
+        pair='ETH/BTC',
+        stake_amount=0.001,
+        fee_open=fee.return_value,
+        open_date=arrow.utcnow().shift(hours=-2).datetime,
+        amount=10,
+        fee_close=fee.return_value,
+        exchange='bittrex',
+        open_rate=1,
+        max_rate=1,
+    )
+    fee_cost = 0.15
+    fee_currency = 'BTC'
+    fee_rate = 0.0075
+    assert trade.fee_open_currency is None
+    assert not trade.fee_updated('buy')
+    assert not trade.fee_updated('sell')
+
+    trade.update_fee(fee_cost, fee_currency, fee_rate, 'buy')
+    assert trade.fee_updated('buy')
+    assert not trade.fee_updated('sell')
+    assert trade.fee_open_currency == fee_currency
+    assert trade.fee_open_cost == fee_cost
+    assert trade.fee_open == fee_rate
+    # Setting buy rate should "guess" close rate
+    assert trade.fee_close == fee_rate
+    assert trade.fee_close_currency is None
+    assert trade.fee_close_cost is None
+
+    fee_rate = 0.0076
+    trade.update_fee(fee_cost, fee_currency, fee_rate, 'sell')
+    assert trade.fee_updated('buy')
+    assert trade.fee_updated('sell')
+    assert trade.fee_close == 0.0076
+    assert trade.fee_close_cost == fee_cost
+    assert trade.fee_close == fee_rate
+
+
+def test_fee_updated(fee):
+    trade = Trade(
+        pair='ETH/BTC',
+        stake_amount=0.001,
+        fee_open=fee.return_value,
+        open_date=arrow.utcnow().shift(hours=-2).datetime,
+        amount=10,
+        fee_close=fee.return_value,
+        exchange='bittrex',
+        open_rate=1,
+        max_rate=1,
+    )
+
+    assert trade.fee_open_currency is None
+    assert not trade.fee_updated('buy')
+    assert not trade.fee_updated('sell')
+    assert not trade.fee_updated('asdf')
+
+    trade.update_fee(0.15, 'BTC', 0.0075, 'buy')
+    assert trade.fee_updated('buy')
+    assert not trade.fee_updated('sell')
+    assert trade.fee_open_currency is not None
+    assert trade.fee_close_currency is None
+
+    trade.update_fee(0.15, 'ABC', 0.0075, 'sell')
+    assert trade.fee_updated('buy')
+    assert trade.fee_updated('sell')
+    assert not trade.fee_updated('asfd')
+
+
 @pytest.mark.usefixtures("init_persistence")
 def test_total_open_trades_stakes(fee):