Merge branch 'develop' into pr/silvavn/3745

config.json.example

@@ -7,7 +7,6 @@
     "timeframe": "5m",
     "dry_run": false,
     "cancel_open_orders_on_exit": false,
-    "trailing_stop": false,
     "unfilledtimeout": {
         "buy": 10,
         "sell": 30

config_binance.json.example

@@ -7,7 +7,6 @@
     "timeframe": "5m",
     "dry_run": true,
     "cancel_open_orders_on_exit": false,
-    "trailing_stop": false,
     "unfilledtimeout": {
         "buy": 10,
         "sell": 30

config_kraken.json.example

@@ -7,7 +7,6 @@
     "timeframe": "5m",
     "dry_run": true,
     "cancel_open_orders_on_exit": false,
-    "trailing_stop": false,
     "unfilledtimeout": {
         "buy": 10,
         "sell": 30

docs/bot-usage.md

@@ -5,6 +5,9 @@ This page explains the different parameters of the bot and how to run it.
 !!! Note
     If you've used `setup.sh`, don't forget to activate your virtual environment (`source .env/bin/activate`) before running freqtrade commands.
 
+!!! Warning "Up-to-date clock"
+    The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges.
+
 ## Bot commands
 
 ```

docs/data-download.md

@@ -15,61 +15,91 @@ Otherwise `--exchange` becomes mandatory.
 ### Usage
 
 ```
-usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]]
+usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
-                               [--pairs-file FILE] [--days INT] [--dl-trades] [--exchange EXCHANGE]
+                               [-d PATH] [--userdir PATH]
+                               [-p PAIRS [PAIRS ...]] [--pairs-file FILE]
+                               [--days INT] [--dl-trades]
+                               [--exchange EXCHANGE]
                                [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
-                               [--erase] [--data-format-ohlcv {json,jsongz}] [--data-format-trades {json,jsongz}]
+                               [--erase]
+                               [--data-format-ohlcv {json,jsongz,hdf5}]
+                               [--data-format-trades {json,jsongz,hdf5}]
 
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
-                        Show profits for only these pairs. Pairs are space-separated.
+                        Show profits for only these pairs. Pairs are space-
+                        separated.
   --pairs-file FILE     File containing a list of pairs to download.
   --days INT            Download data for given number of days.
-  --dl-trades           Download trades instead of OHLCV data. The bot will resample trades to the desired timeframe as specified as
+  --dl-trades           Download trades instead of OHLCV data. The bot will
-                        --timeframes/-t.
+                        resample trades to the desired timeframe as specified
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no config is provided.
+                        as --timeframes/-t.
+  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
+                        config is provided.
   -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
-                        Specify which tickers to download. Space-separated list. Default: `1m 5m`.
+                        Specify which tickers to download. Space-separated
-  --erase               Clean all existing data for the selected exchange/pairs/timeframes.
+                        list. Default: `1m 5m`.
-  --data-format-ohlcv {json,jsongz}
+  --erase               Clean all existing data for the selected
-                        Storage format for downloaded candle (OHLCV) data. (default: `json`).
+                        exchange/pairs/timeframes.
-  --data-format-trades {json,jsongz}
+  --data-format-ohlcv {json,jsongz,hdf5}
-                        Storage format for downloaded trades data. (default: `jsongz`).
+                        Storage format for downloaded candle (OHLCV) data.
+                        (default: `json`).
+  --data-format-trades {json,jsongz,hdf5}
+                        Storage format for downloaded trades data. (default:
+                        `jsongz`).
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details.
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
   -V, --version         show program's version number and exit
   -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-`
+                        Specify configuration file (default:
-                        to read config from stdin.
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
   -d PATH, --datadir PATH
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
+
 ```
 
 ### Data format
 
-Freqtrade currently supports 2 dataformats, `json` (plain "text" json files) and `jsongz` (a gzipped version of json files).
+Freqtrade currently supports 3 data-formats for both OHLCV and trades data:
+
+* `json` (plain "text" json files)
+* `jsongz` (a gzip-zipped version of json files)
+* `hdf5` (a high performance datastore)
+
 By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data.
 
-This can be changed via the `--data-format-ohlcv` and `--data-format-trades` parameters respectivly.
+This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively.
+To persist this change, you can should also add the following snippet to your configuration, so you don't have to insert the above arguments each time:
 
-If the default dataformat has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well.
+``` jsonc
+    // ...
+    "dataformat_ohlcv": "hdf5",
+    "dataformat_trades": "hdf5",
+    // ...
+```
+
+If the default data-format has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well.
 
 !!! Note
-    You can convert between data-formats using the [convert-data](#subcommand-convert-data) and [convert-trade-data](#subcommand-convert-trade-data) methods.
+    You can convert between data-formats using the [convert-data](#sub-command-convert-data) and [convert-trade-data](#sub-command-convert-trade-data) methods.
 
-#### Subcommand convert data
+#### Sub-command convert data
 
 ```
 usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                               [-d PATH] [--userdir PATH]
                               [-p PAIRS [PAIRS ...]] --format-from
-                              {json,jsongz} --format-to {json,jsongz}
+                              {json,jsongz,hdf5} --format-to
-                              [--erase]
+                              {json,jsongz,hdf5} [--erase]
                               [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
 
 optional arguments:
@@ -77,9 +107,9 @@ optional arguments:
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Show profits for only these pairs. Pairs are space-
                         separated.
-  --format-from {json,jsongz}
+  --format-from {json,jsongz,hdf5}
                         Source format for data conversion.
-  --format-to {json,jsongz}
+  --format-to {json,jsongz,hdf5}
                         Destination format for data conversion.
   --erase               Clean all existing data for the selected
                         exchange/pairs/timeframes.
@@ -94,9 +124,10 @@ Common arguments:
                         details.
   -V, --version         show program's version number and exit
   -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`).
+                        Specify configuration file (default:
-                        Multiple --config options may be used. Can be set to
+                        `userdir/config.json` or `config.json` whichever
-                        `-` to read config from stdin.
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
   -d PATH, --datadir PATH
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
@@ -112,23 +143,23 @@ It'll also remove original json data files (`--erase` parameter).
 freqtrade convert-data --format-from json --format-to jsongz --datadir ~/.freqtrade/data/binance -t 5m 15m --erase
 ```
 
-#### Subcommand convert-trade data
+#### Sub-command convert trade data
 
 ```
 usage: freqtrade convert-trade-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                     [-d PATH] [--userdir PATH]
                                     [-p PAIRS [PAIRS ...]] --format-from
-                                    {json,jsongz} --format-to {json,jsongz}
+                                    {json,jsongz,hdf5} --format-to
-                                    [--erase]
+                                    {json,jsongz,hdf5} [--erase]
 
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Show profits for only these pairs. Pairs are space-
                         separated.
-  --format-from {json,jsongz}
+  --format-from {json,jsongz,hdf5}
                         Source format for data conversion.
-  --format-to {json,jsongz}
+  --format-to {json,jsongz,hdf5}
                         Destination format for data conversion.
   --erase               Clean all existing data for the selected
                         exchange/pairs/timeframes.
@@ -140,13 +171,15 @@ Common arguments:
                         details.
   -V, --version         show program's version number and exit
   -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`).
+                        Specify configuration file (default:
-                        Multiple --config options may be used. Can be set to
+                        `userdir/config.json` or `config.json` whichever
-                        `-` to read config from stdin.
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
   -d PATH, --datadir PATH
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
+
 ```
 
 ##### Example converting trades
@@ -158,21 +191,21 @@ It'll also remove original jsongz data files (`--erase` parameter).
 freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase
 ```
 
-### Subcommand list-data
+### Sub-command list-data
 
-You can get a list of downloaded data using the `list-data` subcommand.
+You can get a list of downloaded data using the `list-data` sub-command.
 
 ```
 usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                            [--userdir PATH] [--exchange EXCHANGE]
-                           [--data-format-ohlcv {json,jsongz}]
+                           [--data-format-ohlcv {json,jsongz,hdf5}]
                            [-p PAIRS [PAIRS ...]]
 
 optional arguments:
   -h, --help            show this help message and exit
   --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
                         config is provided.
-  --data-format-ohlcv {json,jsongz}
+  --data-format-ohlcv {json,jsongz,hdf5}
                         Storage format for downloaded candle (OHLCV) data.
                         (default: `json`).
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
@@ -194,6 +227,7 @@ Common arguments:
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
+
 ```
 
 #### Example list-data
@@ -249,7 +283,7 @@ This will download historical candle (OHLCV) data for all the currency pairs you
 ### Other Notes
 
 - To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`.
-- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust ratelimits etc.)
+- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.)
 - To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`.
 - To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days).
 - Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data.
@@ -257,7 +291,7 @@ This will download historical candle (OHLCV) data for all the currency pairs you
 
 ### Trades (tick) data
 
-By default, `download-data` subcommand downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
+By default, `download-data` sub-command downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
 This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes.
 
 Since this data is large by default, the files use gzip by default. They are stored in your data-directory with the naming convention of `<pair>-trades.json.gz` (`ETH_BTC-trades.json.gz`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.

docs/deprecated.md

@@ -32,4 +32,4 @@ The old section of configuration parameters (`"pairlist"`) has been deprecated i
 
 ### deprecation of bidVolume and askVolume from volume-pairlist
 
-Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4.
+Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4, and have been removed in 2020.9.

docs/developer.md

@@ -10,6 +10,15 @@ Documentation is available at [https://freqtrade.io](https://www.freqtrade.io/)
 
 Special fields for the documentation (like Note boxes, ...) can be found [here](https://squidfunk.github.io/mkdocs-material/extensions/admonition/).
 
+To test the documentation locally use the following commands.
+
+``` bash
+pip install -r docs/requirements-docs.txt
+mkdocs serve
+```
+
+This will spin up a local server (usually on port 8000) so you can see if everything looks as you'd like it to.
+
 ## Developer setup
 
 To configure a development environment, best use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
@@ -52,6 +61,7 @@ The fastest and easiest way to start up is to use docker-compose.develop which g
 * [docker-compose](https://docs.docker.com/compose/install/)
 
 #### Starting the bot
+
 ##### Use the develop dockerfile
 
 ``` bash
@@ -74,7 +84,7 @@ docker-compose up
 docker-compose build
 ```
 
-##### Execing (effectively SSH into the container)
+##### Executing (effectively SSH into the container)
 
 The `exec` command requires that the container already be running, if you want to start it
 that can be effected by `docker-compose up` or `docker-compose run freqtrade_develop`
@@ -127,7 +137,7 @@ First of all, have a look at the [VolumePairList](https://github.com/freqtrade/f
 
 This is a simple Handler, which however serves as a good example on how to start developing.
 
-Next, modify the classname of the Handler (ideally align this with the module filename).
+Next, modify the class-name of the Handler (ideally align this with the module filename).
 
 The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`), the pairlist dedicated configuration (`self._pairlistconfig`) and the absolute position within the list of pairlists.
 
@@ -147,7 +157,7 @@ Configuration for the chain of Pairlist Handlers is done in the bot configuratio
 
 By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the pairlist. Please follow this to ensure a consistent user experience.
 
-Additional parameters can be configured as needed. For instance, `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic.
+Additional parameters can be configured as needed. For instance, `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successful and dynamic.
 
 #### short_desc
 
@@ -163,7 +173,7 @@ This is called with each iteration of the bot (only if the Pairlist Handler is a
 
 It must return the resulting pairlist (which may then be passed into the chain of Pairlist Handlers).
 
-Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected.
+Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the end-result is not shorter than expected.
 
 #### filter_pairlist
 
@@ -171,13 +181,13 @@ This method is called for each Pairlist Handler in the chain by the pairlist man
 
 This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations.
 
-It get's passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`.
+It gets passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`.
 
 The default implementation in the base class simply calls the `_validate_pair()` method for each pair in the pairlist, but you may override it. So you should either implement the `_validate_pair()` in your Pairlist Handler or override `filter_pairlist()` to do something else.
 
 If overridden, it must return the resulting pairlist (which may then be passed into the next Pairlist Handler in the chain).
 
-Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected.
+Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the end result is not shorter than expected.
 
 In `VolumePairList`, this implements different methods of sorting, does early validation so only the expected number of pairs is returned.
 
@@ -201,7 +211,7 @@ Most exchanges supported by CCXT should work out of the box.
 
 Check if the new exchange supports Stoploss on Exchange orders through their API.
 
-Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselfs. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects.
+Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselves. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects.
 
 ### Incomplete candles
 
@@ -274,6 +284,7 @@ git checkout -b new_release <commitid>
 
 Determine if crucial bugfixes have been made between this commit and the current state, and eventually cherry-pick these.
 
+* Merge the release branch (master) into this branch.
 * Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi.
 * Commit this part
 * push that branch to the remote and create a PR against the master branch
@@ -281,14 +292,14 @@ Determine if crucial bugfixes have been made between this commit and the current
 ### Create changelog from git commits
 
 !!! Note
-    Make sure that the master branch is uptodate!
+    Make sure that the master branch is up-to-date!
 
 ``` bash
 # Needs to be done before merging / pulling that branch.
 git log --oneline --no-decorate --no-merges master..new_release
 ```
 
-To keep the release-log short, best wrap the full git changelog into a collapsible details secction.
+To keep the release-log short, best wrap the full git changelog into a collapsible details section.
 
 ```markdown
 <details>
@@ -312,6 +323,9 @@ Once the PR against master is merged (best right after merging):
 
 ### pypi
 
+!!! Note
+    This process is now automated as part of Github Actions.
+
 To create a pypi release, please run the following commands:
 
 Additional requirement: `wheel`, `twine` (for uploading), account on pypi with proper permissions.

docs/index.md

@@ -37,13 +37,9 @@ Freqtrade is a crypto-currency algorithmic trading software developed in python
 
 ## Requirements
 
-### Up to date clock
-
-The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges.
-
 ### Hardware requirements
 
-To run this bot we recommend you a cloud instance with a minimum of:
+To run this bot we recommend you a linux cloud instance with a minimum of:
 
 - 2GB RAM
 - 1GB disk space

docs/installation.md

@@ -18,6 +18,9 @@ Click each one for install guide:
 
  We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot), which is optional but recommended.
 
+!!! Warning "Up-to-date clock"
+    The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges.
+
 ## Quick start
 
 Freqtrade provides the Linux/MacOS Easy Installation script to install all dependencies and help you configure the bot.

docs/requirements-docs.txt

@@ -1,2 +1,2 @@
-mkdocs-material==5.5.8
+mkdocs-material==5.5.11
 mdx_truly_sane_lists==1.2

freqtrade/commands/arguments.py

@@ -15,7 +15,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
 
 ARGS_TRADE = ["db_url", "sd_notify", "dry_run"]
 
-ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange",
+ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
                         "max_open_trades", "stake_amount", "fee"]
 
 ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",

freqtrade/constants.py

@@ -24,7 +24,7 @@ ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
                        'AgeFilter', 'PrecisionFilter', 'PriceFilter',
                        'ShuffleFilter', 'SpreadFilter']
-AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
+AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
 DRY_RUN_WALLET = 1000
 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons
@@ -338,9 +338,12 @@ SCHEMA_MINIMAL_REQUIRED = [
 
 CANCEL_REASON = {
     "TIMEOUT": "cancelled due to timeout",
-    "PARTIALLY_FILLED": "partially filled - keeping order open",
+    "PARTIALLY_FILLED_KEEP_OPEN": "partially filled - keeping order open",
+    "PARTIALLY_FILLED": "partially filled",
+    "FULLY_CANCELLED": "fully cancelled",
     "ALL_CANCELLED": "cancelled (all unfilled and partially filled open orders cancelled)",
     "CANCELLED_ON_EXCHANGE": "cancelled on exchange",
+    "FORCE_SELL": "forcesold",
 }
 
 # List of pairs with their timeframes

freqtrade/data/converter.py

@@ -255,6 +255,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
                                   drop_incomplete=False,
                                   startup_candles=0)
             logger.info(f"Converting {len(data)} candles for {pair}")
+            if len(data) > 0:
                 trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data)
                 if erase and convert_from != convert_to:
                     logger.info(f"Deleting source data for {pair} / {timeframe}")

freqtrade/data/dataprovider.py

@@ -39,6 +39,12 @@ class DataProvider:
         """
         self.__cached_pairs[(pair, timeframe)] = (dataframe, Arrow.utcnow().datetime)
 
+    def add_pairlisthandler(self, pairlists) -> None:
+        """
+        Allow adding pairlisthandler after initialization
+        """
+        self._pairlists = pairlists
+
     def refresh(self,
                 pairlist: ListPairsWithTimeframes,
                 helping_pairs: ListPairsWithTimeframes = None) -> None:

freqtrade/data/history/hdf5datahandler.py

@@ -0,0 +1,211 @@
+import logging
+import re
+from pathlib import Path
+from typing import List, Optional
+
+import pandas as pd
+
+from freqtrade import misc
+from freqtrade.configuration import TimeRange
+from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS,
+                                 DEFAULT_TRADES_COLUMNS,
+                                 ListPairsWithTimeframes)
+
+from .idatahandler import IDataHandler, TradeList
+
+logger = logging.getLogger(__name__)
+
+
+class HDF5DataHandler(IDataHandler):
+
+    _columns = DEFAULT_DATAFRAME_COLUMNS
+
+    @classmethod
+    def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Tuples of (pair, timeframe)
+        """
+        _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.h5)', p.name)
+                for p in datadir.glob("*.h5")]
+        return [(match[1].replace('_', '/'), match[2]) for match in _tmp
+                if match and len(match.groups()) > 1]
+
+    @classmethod
+    def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
+        """
+        Returns a list of all pairs with ohlcv data available in this datadir
+        for the specified timeframe
+        :param datadir: Directory to search for ohlcv files
+        :param timeframe: Timeframe to search pairs for
+        :return: List of Pairs
+        """
+
+        _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.h5)', p.name)
+                for p in datadir.glob(f"*{timeframe}.h5")]
+        # Check if regex found something and only return these results
+        return [match[0].replace('_', '/') for match in _tmp if match]
+
+    def ohlcv_store(self, pair: str, timeframe: str, data: pd.DataFrame) -> None:
+        """
+        Store data in hdf5 file.
+        :param pair: Pair - used to generate filename
+        :timeframe: Timeframe - used to generate filename
+        :data: Dataframe containing OHLCV data
+        :return: None
+        """
+        key = self._pair_ohlcv_key(pair, timeframe)
+        _data = data.copy()
+
+        filename = self._pair_data_filename(self._datadir, pair, timeframe)
+
+        ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc')
+        ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date'])
+
+        ds.close()
+
+    def _ohlcv_load(self, pair: str, timeframe: str,
+                    timerange: Optional[TimeRange] = None) -> pd.DataFrame:
+        """
+        Internal method used to load data for one pair from disk.
+        Implements the loading and conversion to a Pandas dataframe.
+        Timerange trimming and dataframe validation happens outside of this method.
+        :param pair: Pair to load data
+        :param timeframe: Timeframe (e.g. "5m")
+        :param timerange: Limit data to be loaded to this timerange.
+                        Optionally implemented by subclasses to avoid loading
+                        all data where possible.
+        :return: DataFrame with ohlcv data, or empty DataFrame
+        """
+        key = self._pair_ohlcv_key(pair, timeframe)
+        filename = self._pair_data_filename(self._datadir, pair, timeframe)
+
+        if not filename.exists():
+            return pd.DataFrame(columns=self._columns)
+        where = []
+        if timerange:
+            if timerange.starttype == 'date':
+                where.append(f"date >= Timestamp({timerange.startts * 1e9})")
+            if timerange.stoptype == 'date':
+                where.append(f"date < Timestamp({timerange.stopts * 1e9})")
+
+        pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)
+
+        if list(pairdata.columns) != self._columns:
+            raise ValueError("Wrong dataframe format")
+        pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float',
+                                          'low': 'float', 'close': 'float', 'volume': 'float'})
+        return pairdata
+
+    def ohlcv_purge(self, pair: str, timeframe: str) -> bool:
+        """
+        Remove data for this pair
+        :param pair: Delete data for this pair.
+        :param timeframe: Timeframe (e.g. "5m")
+        :return: True when deleted, false if file did not exist.
+        """
+        filename = self._pair_data_filename(self._datadir, pair, timeframe)
+        if filename.exists():
+            filename.unlink()
+            return True
+        return False
+
+    def ohlcv_append(self, pair: str, timeframe: str, data: pd.DataFrame) -> None:
+        """
+        Append data to existing data structures
+        :param pair: Pair
+        :param timeframe: Timeframe this ohlcv data is for
+        :param data: Data to append.
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    def trades_get_pairs(cls, datadir: Path) -> List[str]:
+        """
+        Returns a list of all pairs for which trade data is available in this
+        :param datadir: Directory to search for ohlcv files
+        :return: List of Pairs
+        """
+        _tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name)
+                for p in datadir.glob("*trades.h5")]
+        # Check if regex found something and only return these results to avoid exceptions.
+        return [match[0].replace('_', '/') for match in _tmp if match]
+
+    def trades_store(self, pair: str, data: TradeList) -> None:
+        """
+        Store trades data (list of Dicts) to file
+        :param pair: Pair - used for filename
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
+        """
+        key = self._pair_trades_key(pair)
+
+        ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair),
+                         mode='a', complevel=9, complib='blosc')
+        ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS),
+               format='table', data_columns=['timestamp'])
+        ds.close()
+
+    def trades_append(self, pair: str, data: TradeList):
+        """
+        Append data to existing files
+        :param pair: Pair - used for filename
+        :param data: List of Lists containing trade data,
+                     column sequence as in DEFAULT_TRADES_COLUMNS
+        """
+        raise NotImplementedError()
+
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+        """
+        Load a pair from h5 file.
+        :param pair: Load trades for this pair
+        :param timerange: Timerange to load trades for - currently not implemented
+        :return: List of trades
+        """
+        key = self._pair_trades_key(pair)
+        filename = self._pair_trades_filename(self._datadir, pair)
+
+        if not filename.exists():
+            return []
+        where = []
+        if timerange:
+            if timerange.starttype == 'date':
+                where.append(f"timestamp >= {timerange.startts * 1e3}")
+            if timerange.stoptype == 'date':
+                where.append(f"timestamp < {timerange.stopts * 1e3}")
+
+        trades = pd.read_hdf(filename, key=key, mode="r", where=where)
+        return trades.values.tolist()
+
+    def trades_purge(self, pair: str) -> bool:
+        """
+        Remove data for this pair
+        :param pair: Delete data for this pair.
+        :return: True when deleted, false if file did not exist.
+        """
+        filename = self._pair_trades_filename(self._datadir, pair)
+        if filename.exists():
+            filename.unlink()
+            return True
+        return False
+
+    @classmethod
+    def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str:
+        return f"{pair}/ohlcv/tf_{timeframe}"
+
+    @classmethod
+    def _pair_trades_key(cls, pair: str) -> str:
+        return f"{pair}/trades"
+
+    @classmethod
+    def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path:
+        pair_s = misc.pair_to_filename(pair)
+        filename = datadir.joinpath(f'{pair_s}-{timeframe}.h5')
+        return filename
+
+    @classmethod
+    def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
+        pair_s = misc.pair_to_filename(pair)
+        filename = datadir.joinpath(f'{pair_s}-trades.h5')
+        return filename

freqtrade/data/history/history_utils.py

@@ -9,7 +9,8 @@ from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
-from freqtrade.data.converter import (ohlcv_to_dataframe,
+from freqtrade.data.converter import (clean_ohlcv_dataframe,
+                                      ohlcv_to_dataframe,
                                       trades_remove_duplicates,
                                       trades_to_ohlcv)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
@@ -202,7 +203,10 @@ def _download_pair_history(datadir: Path,
         if data.empty:
             data = new_dataframe
         else:
-            data = data.append(new_dataframe)
+            # Run cleaning again to ensure there were no duplicate candles
+            # Especially between existing and new data.
+            data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair,
+                                         fill_missing=False, drop_incomplete=False)
 
         logger.debug("New  Start: %s",
                      f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')

freqtrade/data/history/idatahandler.py

@@ -50,9 +50,7 @@ class IDataHandler(ABC):
     @abstractmethod
     def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None:
         """
-        Store data in json format "values".
+        Store ohlcv data.
-            format looks as follows:
-            [[<date>,<open>,<high>,<low>,<close>]]
         :param pair: Pair - used to generate filename
         :timeframe: Timeframe - used to generate filename
         :data: Dataframe containing OHLCV data
@@ -239,6 +237,9 @@ def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
     elif datatype == 'jsongz':
         from .jsondatahandler import JsonGzDataHandler
         return JsonGzDataHandler
+    elif datatype == 'hdf5':
+        from .hdf5datahandler import HDF5DataHandler
+        return HDF5DataHandler
     else:
         raise ValueError(f"No datahandler for datatype {datatype} available.")
 

freqtrade/exchange/exchange.py

@@ -973,6 +973,11 @@ class Exchange:
     @retrier
     def cancel_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
+            order = self._dry_run_open_orders.get(order_id)
+            if order:
+                order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']})
+                return order
+            else:
                 return {}
 
         try:

freqtrade/freqtradebot.py

@@ -618,7 +618,7 @@ class FreqtradeBot:
         # Send the message
         self.rpc.send_msg(msg)
 
-    def _notify_buy_cancel(self, trade: Trade, order_type: str) -> None:
+    def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
         """
         Sends rpc notification when a buy cancel occured.
         """
@@ -637,6 +637,7 @@ class FreqtradeBot:
             'amount': trade.amount,
             'open_date': trade.open_date,
             'current_rate': current_rate,
+            'reason': reason,
         }
 
         # Send the message
@@ -835,7 +836,7 @@ class FreqtradeBot:
             stop_price = trade.open_rate * (1 + stoploss)
 
             if self.create_stoploss_order(trade=trade, stop_price=stop_price):
-                trade.stoploss_last_update = datetime.now()
+                trade.stoploss_last_update = datetime.utcnow()
                 return False
 
         # If stoploss order is canceled for some reason we add it
@@ -974,7 +975,6 @@ class FreqtradeBot:
 
         # Cancelled orders may have the status of 'canceled' or 'closed'
         if order['status'] not in ('canceled', 'closed'):
-            reason = constants.CANCEL_REASON['TIMEOUT']
             corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
                                                             trade.amount)
             # Avoid race condition where the order could not be cancelled coz its already filled.
@@ -992,13 +992,13 @@ class FreqtradeBot:
 
         # Using filled to determine the filled amount
         filled_amount = safe_value_fallback2(corder, order, 'filled', 'filled')
-
         if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
             logger.info('Buy order fully cancelled. Removing %s from database.', trade)
             # if trade is not partially completed, just delete the trade
             Trade.session.delete(trade)
             Trade.session.flush()
             was_trade_fully_canceled = True
+            reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
         else:
             # if trade is partially complete, edit the stake details for the trade
             # and close the order
@@ -1011,13 +1011,11 @@ class FreqtradeBot:
 
             trade.open_order_id = None
             logger.info('Partial buy order timeout for %s.', trade)
-            self.rpc.send_msg({
+            reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}"
-                'type': RPCMessageType.STATUS_NOTIFICATION,
-                'status': f'Remaining buy order for {trade.pair} cancelled due to timeout'
-            })
 
         self.wallets.update()
-        self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'])
+        self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'],
+                                reason=reason)
         return was_trade_fully_canceled
 
     def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str:
@@ -1048,7 +1046,7 @@ class FreqtradeBot:
             trade.open_order_id = None
         else:
             # TODO: figure out how to handle partially complete sell orders
-            reason = constants.CANCEL_REASON['PARTIALLY_FILLED']
+            reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
 
         self.wallets.update()
         self._notify_sell_cancel(

freqtrade/optimize/backtesting.py

@@ -96,6 +96,7 @@ class Backtesting:
                 "PrecisionFilter not allowed for backtesting multiple strategies."
             )
 
+        dataprovider.add_pairlisthandler(self.pairlists)
         self.pairlists.refresh_pairlist()
 
         if len(self.pairlists.whitelist) == 0:

freqtrade/pairlist/VolumePairList.py

@@ -14,7 +14,7 @@ from freqtrade.pairlist.IPairList import IPairList
 logger = logging.getLogger(__name__)
 
 
-SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume']
+SORT_VALUES = ['quoteVolume']
 
 
 class VolumePairList(IPairList):
@@ -45,11 +45,6 @@ class VolumePairList(IPairList):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
 
-        if self._sort_key != 'quoteVolume':
-            logger.warning(
-                "DEPRECATED: using any key other than quoteVolume for VolumePairList is deprecated."
-                )
-
     @property
     def needstickers(self) -> bool:
         """

freqtrade/rpc/rpc.py

@@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 import arrow
 from numpy import NAN, mean
 
+from freqtrade.constants import CANCEL_REASON
 from freqtrade.exceptions import ExchangeError, PricingError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.loggers import bufferHandler
@@ -223,7 +224,8 @@ class RPC:
                 Trade.close_date >= profitday,
                 Trade.close_date < (profitday + timedelta(days=1))
             ]).order_by(Trade.close_date).all()
-            curdayprofit = sum(trade.close_profit_abs for trade in trades)
+            curdayprofit = sum(
+                trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None)
             profit_days[profitday] = {
                 'amount': curdayprofit,
                 'trades': len(trades)
@@ -453,26 +455,19 @@ class RPC:
         """
         def _exec_forcesell(trade: Trade) -> None:
             # Check if there is there is an open order
+            fully_canceled = False
             if trade.open_order_id:
                 order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
 
-                # Cancel open LIMIT_BUY orders and close trade
+                if order['side'] == 'buy':
-                if order and order['status'] == 'open' \
+                    fully_canceled = self._freqtrade.handle_cancel_buy(
-                        and order['type'] == 'limit' \
+                        trade, order, CANCEL_REASON['FORCE_SELL'])
-                        and order['side'] == 'buy':
-                    self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair)
-                    trade.close(order.get('price') or trade.open_rate)
-                    # Do the best effort, if we don't know 'filled' amount, don't try selling
-                    if order['filled'] is None:
-                        return
-                    trade.amount = order['filled']
 
-                # Ignore trades with an attached LIMIT_SELL order
+                if order['side'] == 'sell':
-                if order and order['status'] == 'open' \
+                    # Cancel order - so it is placed anew with a fresh price.
-                        and order['type'] == 'limit' \
+                    self._freqtrade.handle_cancel_sell(trade, order, CANCEL_REASON['FORCE_SELL'])
-                        and order['side'] == 'sell':
-                    return
 
+            if not fully_canceled:
                 # Get current rate and execute sell
                 current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
                 self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL)

freqtrade/rpc/telegram.py

@@ -151,7 +151,7 @@ class Telegram(RPC):
 
         elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION:
             message = ("\N{WARNING SIGN} *{exchange}:* "
-                       "Cancelling Open Buy Order for {pair}".format(**msg))
+                       "Cancelling open buy Order for {pair}. Reason: {reason}.".format(**msg))
 
         elif msg['type'] == RPCMessageType.SELL_NOTIFICATION:
             msg['amount'] = round(msg['amount'], 8)

requirements-common.txt

@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.33.52
+ccxt==1.33.72
 SQLAlchemy==1.3.19
 python-telegram-bot==12.8
 arrow==0.16.0
@@ -13,6 +13,8 @@ TA-Lib==0.4.18
 tabulate==0.8.7
 pycoingecko==1.3.0
 jinja2==2.11.2
+tables==3.6.1
+blosc==1.9.1
 
 # find first, C search in arrays
 py_find_1st==1.1.4
@@ -26,10 +28,10 @@ sdnotify==0.3.2
 # Api server
 flask==1.1.2
 flask-jwt-extended==3.24.1
-flask-cors==3.0.8
+flask-cors==3.0.9
 
 # Support for colorized terminal output
 colorama==0.4.3
 # Building config files interactively
 questionary==1.5.2
-prompt-toolkit==3.0.6
+prompt-toolkit==3.0.7

requirements-dev.txt

@@ -11,7 +11,7 @@ mypy==0.782
 pytest==6.0.1
 pytest-asyncio==0.14.0
 pytest-cov==2.10.1
-pytest-mock==3.3.0
+pytest-mock==3.3.1
 pytest-random-order==1.0.4
 
 # Convert jupyter notebooks to markdown documents

requirements-hyperopt.txt

@@ -7,4 +7,4 @@ scikit-learn==0.23.1
 scikit-optimize==0.7.4
 filelock==3.0.12
 joblib==0.16.0
-progressbar2==3.51.4
+progressbar2==3.52.1

setup.py

@@ -85,6 +85,8 @@ setup(name='freqtrade',
           # from requirements.txt
           'numpy',
           'pandas',
+          'tables',
+          'blosc',
       ],
       extras_require={
           'api': api,

tests/data/test_history.py

@@ -12,7 +12,9 @@ from pandas import DataFrame
 from pandas.testing import assert_frame_equal
 
 from freqtrade.configuration import TimeRange
+from freqtrade.constants import AVAILABLE_DATAHANDLERS
 from freqtrade.data.converter import ohlcv_to_dataframe
+from freqtrade.data.history.hdf5datahandler import HDF5DataHandler
 from freqtrade.data.history.history_utils import (
     _download_pair_history, _download_trades_history,
     _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange,
@@ -620,7 +622,7 @@ def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog):
     _clean_test_file(file5)
 
 
-def test_jsondatahandler_ohlcv_get_pairs(testdatadir):
+def test_datahandler_ohlcv_get_pairs(testdatadir):
     pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m')
     # Convert to set to avoid failures due to sorting
     assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC',
@@ -630,8 +632,11 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir):
     pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m')
     assert set(pairs) == {'UNITTEST/BTC'}
 
+    pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m')
+    assert set(pairs) == {'UNITTEST/BTC'}
 
-def test_jsondatahandler_ohlcv_get_available_data(testdatadir):
+
+def test_datahandler_ohlcv_get_available_data(testdatadir):
     paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir)
     # Convert to set to avoid failures due to sorting
     assert set(paircombs) == {('UNITTEST/BTC', '5m'), ('ETH/BTC', '5m'), ('XLM/BTC', '5m'),
@@ -643,6 +648,8 @@ def test_jsondatahandler_ohlcv_get_available_data(testdatadir):
 
     paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir)
     assert set(paircombs) == {('UNITTEST/BTC', '8m')}
+    paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir)
+    assert set(paircombs) == {('UNITTEST/BTC', '5m')}
 
 
 def test_jsondatahandler_trades_get_pairs(testdatadir):
@@ -653,15 +660,17 @@ def test_jsondatahandler_trades_get_pairs(testdatadir):
 
 def test_jsondatahandler_ohlcv_purge(mocker, testdatadir):
     mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    mocker.patch.object(Path, "unlink", MagicMock())
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
     dh = JsonGzDataHandler(testdatadir)
     assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m')
+    assert unlinkmock.call_count == 0
 
     mocker.patch.object(Path, "exists", MagicMock(return_value=True))
     assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m')
+    assert unlinkmock.call_count == 1
 
 
-def test_jsondatahandler_trades_load(mocker, testdatadir, caplog):
+def test_jsondatahandler_trades_load(testdatadir, caplog):
     dh = JsonGzDataHandler(testdatadir)
     logmsg = "Old trades format detected - converting"
     dh.trades_load('XRP/ETH')
@@ -674,26 +683,144 @@ def test_jsondatahandler_trades_load(mocker, testdatadir, caplog):
 
 def test_jsondatahandler_trades_purge(mocker, testdatadir):
     mocker.patch.object(Path, "exists", MagicMock(return_value=False))
-    mocker.patch.object(Path, "unlink", MagicMock())
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
     dh = JsonGzDataHandler(testdatadir)
     assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 0
 
     mocker.patch.object(Path, "exists", MagicMock(return_value=True))
     assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 1
 
 
-def test_jsondatahandler_ohlcv_append(testdatadir):
+@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
-    dh = JsonGzDataHandler(testdatadir)
+def test_datahandler_ohlcv_append(datahandler, testdatadir, ):
+    dh = get_datahandler(testdatadir, datahandler)
     with pytest.raises(NotImplementedError):
         dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame())
 
 
-def test_jsondatahandler_trades_append(testdatadir):
+@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS)
-    dh = JsonGzDataHandler(testdatadir)
+def test_datahandler_trades_append(datahandler, testdatadir):
+    dh = get_datahandler(testdatadir, datahandler)
     with pytest.raises(NotImplementedError):
         dh.trades_append('UNITTEST/ETH', [])
 
 
+def test_hdf5datahandler_trades_get_pairs(testdatadir):
+    pairs = HDF5DataHandler.trades_get_pairs(testdatadir)
+    # Convert to set to avoid failures due to sorting
+    assert set(pairs) == {'XRP/ETH'}
+
+
+def test_hdf5datahandler_trades_load(testdatadir):
+    dh = HDF5DataHandler(testdatadir)
+    trades = dh.trades_load('XRP/ETH')
+    assert isinstance(trades, list)
+
+    trades1 = dh.trades_load('UNITTEST/NONEXIST')
+    assert trades1 == []
+    # data goes from 2019-10-11 - 2019-10-13
+    timerange = TimeRange.parse_timerange('20191011-20191012')
+
+    trades2 = dh._trades_load('XRP/ETH', timerange)
+    assert len(trades) > len(trades2)
+
+    # unfiltered load has trades before starttime
+    assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0
+    # filtered list does not have trades before starttime
+    assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0
+    # unfiltered load has trades after endtime
+    assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0
+    # filtered list does not have trades after endtime
+    assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0
+
+
+def test_hdf5datahandler_trades_store(testdatadir):
+    dh = HDF5DataHandler(testdatadir)
+    trades = dh.trades_load('XRP/ETH')
+
+    dh.trades_store('XRP/NEW', trades)
+    file = testdatadir / 'XRP_NEW-trades.h5'
+    assert file.is_file()
+    # Load trades back
+    trades_new = dh.trades_load('XRP/NEW')
+
+    assert len(trades_new) == len(trades)
+    assert trades[0][0] == trades_new[0][0]
+    assert trades[0][1] == trades_new[0][1]
+    # assert trades[0][2] == trades_new[0][2]  # This is nan - so comparison does not make sense
+    assert trades[0][3] == trades_new[0][3]
+    assert trades[0][4] == trades_new[0][4]
+    assert trades[0][5] == trades_new[0][5]
+    assert trades[0][6] == trades_new[0][6]
+    assert trades[-1][0] == trades_new[-1][0]
+    assert trades[-1][1] == trades_new[-1][1]
+    # assert trades[-1][2] == trades_new[-1][2]  # This is nan - so comparison does not make sense
+    assert trades[-1][3] == trades_new[-1][3]
+    assert trades[-1][4] == trades_new[-1][4]
+    assert trades[-1][5] == trades_new[-1][5]
+    assert trades[-1][6] == trades_new[-1][6]
+
+    _clean_test_file(file)
+
+
+def test_hdf5datahandler_trades_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = HDF5DataHandler(testdatadir)
+    assert not dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.trades_purge('UNITTEST/NONEXIST')
+    assert unlinkmock.call_count == 1
+
+
+def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir):
+    dh = HDF5DataHandler(testdatadir)
+    ohlcv = dh.ohlcv_load('UNITTEST/BTC', '5m')
+    assert isinstance(ohlcv, DataFrame)
+    assert len(ohlcv) > 0
+
+    file = testdatadir / 'UNITTEST_NEW-5m.h5'
+    assert not file.is_file()
+
+    dh.ohlcv_store('UNITTEST/NEW', '5m', ohlcv)
+    assert file.is_file()
+
+    assert not ohlcv[ohlcv['date'] < '2018-01-15'].empty
+
+    # Data gores from 2018-01-10 - 2018-01-30
+    timerange = TimeRange.parse_timerange('20180115-20180119')
+
+    # Call private function to ensure timerange is filtered in hdf5
+    ohlcv = dh._ohlcv_load('UNITTEST/BTC', '5m', timerange)
+    ohlcv1 = dh._ohlcv_load('UNITTEST/NEW', '5m', timerange)
+    assert len(ohlcv) == len(ohlcv1)
+    assert ohlcv.equals(ohlcv1)
+    assert ohlcv[ohlcv['date'] < '2018-01-15'].empty
+    assert ohlcv[ohlcv['date'] > '2018-01-19'].empty
+
+    _clean_test_file(file)
+
+    # Try loading inexisting file
+    ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', '5m')
+    assert ohlcv.empty
+
+
+def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
+    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
+    unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
+    dh = HDF5DataHandler(testdatadir)
+    assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m')
+    assert unlinkmock.call_count == 0
+
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m')
+    assert unlinkmock.call_count == 1
+
+
 def test_gethandlerclass():
     cl = get_datahandlerclass('json')
     assert cl == JsonDataHandler
@@ -702,6 +829,9 @@ def test_gethandlerclass():
     assert cl == JsonGzDataHandler
     assert issubclass(cl, IDataHandler)
     assert issubclass(cl, JsonDataHandler)
+    cl = get_datahandlerclass('hdf5')
+    assert cl == HDF5DataHandler
+    assert issubclass(cl, IDataHandler)
     with pytest.raises(ValueError, match=r"No datahandler for .*"):
         get_datahandlerclass('DeadBeef')
 
@@ -713,3 +843,6 @@ def test_get_datahandler(testdatadir):
     assert type(dh) == JsonGzDataHandler
     dh1 = get_datahandler(testdatadir, 'jsongz', dh)
     assert id(dh1) == id(dh)
+
+    dh = get_datahandler(testdatadir, 'hdf5')
+    assert type(dh) == HDF5DataHandler

tests/exchange/test_exchange.py

@@ -1761,6 +1761,14 @@ def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
     assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {}
     assert exchange.cancel_stoploss_order(order_id='123', pair='TKN/BTC') == {}
 
+    order = exchange.buy('ETH/BTC', 'limit', 5, 0.55, 'gtc')
+
+    cancel_order = exchange.cancel_order(order_id=order['id'], pair='ETH/BTC')
+    assert order['id'] == cancel_order['id']
+    assert order['amount'] == cancel_order['amount']
+    assert order['pair'] == cancel_order['pair']
+    assert cancel_order['status'] == 'canceled'
+
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 @pytest.mark.parametrize("order,result", [

tests/optimize/test_backtesting.py

@@ -359,6 +359,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
     ]
     for line in exists:
         assert log_has(line, caplog)
+    assert backtesting.strategy.dp._pairlists is not None
 
 
 def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None:

tests/pairlist/test_pairlist.py

@@ -231,9 +231,6 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     # VolumePairList only
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
      "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']),
-    # Different sorting depending on quote or bid volume
-    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}],
-     "BTC",  ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']),
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
      "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']),
     # No pair for ETH, VolumePairList
@@ -263,10 +260,6 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "PrecisionFilter"}],
      "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
-    # Precisionfilter bid
-    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"},
-      {"method": "PrecisionFilter"}],
-     "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']),
     # PriceFilter and VolumePairList
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "PriceFilter", "low_price_ratio": 0.03}],
@@ -293,9 +286,6 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "StaticPairList"}],
      "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']),
     # Static Pairlist before VolumePairList - sorting changes
-    ([{"method": "StaticPairList"},
-      {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}],
-     "BTC", ['HOT/BTC', 'TKN/BTC', 'ETH/BTC']),
     # SpreadFilter
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "SpreadFilter", "max_spread_ratio": 0.005}],
@@ -344,7 +334,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}],
      "BTC", 'filter_at_the_beginning'),  # OperationalException expected
     # Static Pairlist after VolumePairList, on a non-first position
-    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"},
+    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "StaticPairList"}],
         "BTC", 'static_in_the_middle'),
     ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"},

tests/rpc/test_rpc.py

@@ -669,7 +669,8 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
             return_value={
                 'status': 'closed',
                 'type': 'limit',
-                'side': 'buy'
+                'side': 'buy',
+                'filled': 0.0,
             }
         ),
         get_fee=fee,
@@ -695,6 +696,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
     msg = rpc._rpc_forcesell('all')
     assert msg == {'result': 'Created sell orders for all open trades.'}
 
+    freqtradebot.enter_positions()
     msg = rpc._rpc_forcesell('1')
     assert msg == {'result': 'Created sell order for trade 1.'}
 
@@ -707,17 +709,24 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
 
     freqtradebot.state = State.RUNNING
     assert cancel_order_mock.call_count == 0
+    freqtradebot.enter_positions()
     # make an limit-buy open trade
     trade = Trade.query.filter(Trade.id == '1').first()
     filled_amount = trade.amount / 2
+    # Fetch order - it's open first, and closed after cancel_order is called.
     mocker.patch(
         'freqtrade.exchange.Exchange.fetch_order',
-        return_value={
+        side_effect=[{
             'status': 'open',
             'type': 'limit',
             'side': 'buy',
             'filled': filled_amount
-        }
+        }, {
+            'status': 'closed',
+            'type': 'limit',
+            'side': 'buy',
+            'filled': filled_amount
+        }]
     )
     # check that the trade is called, which is done by ensuring exchange.cancel_order is called
     # and trade amount is updated
@@ -725,6 +734,16 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
     assert cancel_order_mock.call_count == 1
     assert trade.amount == filled_amount
 
+    mocker.patch(
+        'freqtrade.exchange.Exchange.fetch_order',
+        return_value={
+            'status': 'open',
+            'type': 'limit',
+            'side': 'buy',
+            'filled': filled_amount
+        })
+
+    freqtradebot.config['max_open_trades'] = 3
     freqtradebot.enter_positions()
     trade = Trade.query.filter(Trade.id == '2').first()
     amount = trade.amount
@@ -744,20 +763,22 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
     assert cancel_order_mock.call_count == 2
     assert trade.amount == amount
 
-    freqtradebot.enter_positions()
     # make an limit-sell open trade
     mocker.patch(
         'freqtrade.exchange.Exchange.fetch_order',
         return_value={
             'status': 'open',
             'type': 'limit',
-            'side': 'sell'
+            'side': 'sell',
+            'amount': amount,
+            'remaining': amount,
+            'filled': 0.0
         }
     )
     msg = rpc._rpc_forcesell('3')
     assert msg == {'result': 'Created sell order for trade 3.'}
     # status quo, no exchange calls
-    assert cancel_order_mock.call_count == 2
+    assert cancel_order_mock.call_count == 3
 
 
 def test_performance_handle(default_conf, ticker, limit_buy_order, fee,

tests/rpc/test_rpc_telegram.py

@@ -14,6 +14,7 @@ from telegram import Chat, Message, Update
 from telegram.error import NetworkError
 
 from freqtrade import __version__
+from freqtrade.constants import CANCEL_REASON
 from freqtrade.edge import PairInfo
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.loggers import setup_logging
@@ -725,7 +726,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee,
     context.args = ["1"]
     telegram._forcesell(update=update, context=context)
 
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
     last_msg = rpc_mock.call_args_list[-1][0][0]
     assert {
         'type': RPCMessageType.SELL_NOTIFICATION,
@@ -784,7 +785,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee,
     context.args = ["1"]
     telegram._forcesell(update=update, context=context)
 
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
 
     last_msg = rpc_mock.call_args_list[-1][0][0]
     assert {
@@ -834,8 +835,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None
     context.args = ["all"]
     telegram._forcesell(update=update, context=context)
 
-    assert rpc_mock.call_count == 4
+    # Called for each trade 3 times
-    msg = rpc_mock.call_args_list[0][0][0]
+    assert rpc_mock.call_count == 8
+    msg = rpc_mock.call_args_list[1][0][0]
     assert {
         'type': RPCMessageType.SELL_NOTIFICATION,
         'trade_id': 1,
@@ -1343,9 +1345,10 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None:
         'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
         'exchange': 'Bittrex',
         'pair': 'ETH/BTC',
+        'reason': CANCEL_REASON['TIMEOUT']
     })
-    assert msg_mock.call_args[0][0] \
+    assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Bittrex:* '
-        == ('\N{WARNING SIGN} *Bittrex:* Cancelling Open Buy Order for ETH/BTC')
+            'Cancelling open buy Order for ETH/BTC. Reason: cancelled due to timeout.')
 
 
 def test_send_msg_sell_notification(default_conf, mocker) -> None:

tests/test_freqtradebot.py

@@ -2289,7 +2289,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old
     # note this is for a partially-complete buy order
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 1
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     assert trades[0].amount == 23.0
@@ -2324,7 +2324,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
     assert log_has_re(r"Applying fee on amount for Trade.*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 1
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -2364,7 +2364,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade,
     assert log_has_re(r"Could not update trade amount: .*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 1
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -2527,13 +2527,15 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None:
     send_msg_mock.reset_mock()
 
     order['amount'] = 2
-    assert freqtrade.handle_cancel_sell(trade, order, reason) == CANCEL_REASON['PARTIALLY_FILLED']
+    assert freqtrade.handle_cancel_sell(trade, order, reason
+                                        ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
     # Assert cancel_order was not called (callcount remains unchanged)
     assert cancel_order_mock.call_count == 1
     assert send_msg_mock.call_count == 1
-    assert freqtrade.handle_cancel_sell(trade, order, reason) == CANCEL_REASON['PARTIALLY_FILLED']
+    assert freqtrade.handle_cancel_sell(trade, order, reason
+                                        ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
     # Message should not be iterated again
-    assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED']
+    assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
     assert send_msg_mock.call_count == 1