Document hdf5 dataformat
This commit is contained in:
Matthias
parent
edb582e522
commit
119bf2a8ea
@ -15,61 +15,91 @@ Otherwise `--exchange` becomes mandatory.
|
||||
### Usage
|
||||
|
||||
```
|
||||
usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]]
|
||||
[--pairs-file FILE] [--days INT] [--dl-trades] [--exchange EXCHANGE]
|
||||
usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
|
||||
[-d PATH] [--userdir PATH]
|
||||
[-p PAIRS [PAIRS ...]] [--pairs-file FILE]
|
||||
[--days INT] [--dl-trades]
|
||||
[--exchange EXCHANGE]
|
||||
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
|
||||
[--erase] [--data-format-ohlcv {json,jsongz}] [--data-format-trades {json,jsongz}]
|
||||
[--erase]
|
||||
[--data-format-ohlcv {json,jsongz,hdf5}]
|
||||
[--data-format-trades {json,jsongz,hdf5}]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
|
||||
Show profits for only these pairs. Pairs are space-separated.
|
||||
Show profits for only these pairs. Pairs are space-
|
||||
separated.
|
||||
--pairs-file FILE File containing a list of pairs to download.
|
||||
--days INT Download data for given number of days.
|
||||
--dl-trades Download trades instead of OHLCV data. The bot will resample trades to the desired timeframe as specified as
|
||||
--timeframes/-t.
|
||||
--exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided.
|
||||
--dl-trades Download trades instead of OHLCV data. The bot will
|
||||
resample trades to the desired timeframe as specified
|
||||
as --timeframes/-t.
|
||||
--exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no
|
||||
config is provided.
|
||||
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
|
||||
Specify which tickers to download. Space-separated list. Default: `1m 5m`.
|
||||
--erase Clean all existing data for the selected exchange/pairs/timeframes.
|
||||
--data-format-ohlcv {json,jsongz}
|
||||
Storage format for downloaded candle (OHLCV) data. (default: `json`).
|
||||
--data-format-trades {json,jsongz}
|
||||
Storage format for downloaded trades data. (default: `jsongz`).
|
||||
Specify which tickers to download. Space-separated
|
||||
list. Default: `1m 5m`.
|
||||
--erase Clean all existing data for the selected
|
||||
exchange/pairs/timeframes.
|
||||
--data-format-ohlcv {json,jsongz,hdf5}
|
||||
Storage format for downloaded candle (OHLCV) data.
|
||||
(default: `json`).
|
||||
--data-format-trades {json,jsongz,hdf5}
|
||||
Storage format for downloaded trades data. (default:
|
||||
`jsongz`).
|
||||
|
||||
Common arguments:
|
||||
-v, --verbose Verbose mode (-vv for more, -vvv to get all messages).
|
||||
--logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details.
|
||||
--logfile FILE Log to the file specified. Special values are:
|
||||
'syslog', 'journald'. See the documentation for more
|
||||
details.
|
||||
-V, --version show program's version number and exit
|
||||
-c PATH, --config PATH
|
||||
Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-`
|
||||
to read config from stdin.
|
||||
Specify configuration file (default:
|
||||
`userdir/config.json` or `config.json` whichever
|
||||
exists). Multiple --config options may be used. Can be
|
||||
set to `-` to read config from stdin.
|
||||
-d PATH, --datadir PATH
|
||||
Path to directory with historical backtesting data.
|
||||
--userdir PATH, --user-data-dir PATH
|
||||
Path to userdata directory.
|
||||
|
||||
```
|
||||
|
||||
### Data format
|
||||
|
||||
Freqtrade currently supports 2 dataformats, `json` (plain "text" json files) and `jsongz` (a gzipped version of json files).
|
||||
Freqtrade currently supports 3 data-formats for both OHLCV and trades data:
|
||||
|
||||
* `json` (plain "text" json files)
|
||||
* `jsongz` (a gzip-zipped version of json files)
|
||||
* `hdf5` (a high performance datastore)
|
||||
|
||||
By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data.
|
||||
|
||||
This can be changed via the `--data-format-ohlcv` and `--data-format-trades` parameters respectivly.
|
||||
This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively.
|
||||
To persist this change, you can should also add the following snippet to your configuration, so you don't have to insert the above arguments each time:
|
||||
|
||||
If the default dataformat has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well.
|
||||
``` jsonc
|
||||
// ...
|
||||
"dataformat_ohlcv": "hdf5",
|
||||
"dataformat_trades": "hdf5",
|
||||
// ...
|
||||
```
|
||||
|
||||
If the default data-format has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well.
|
||||
|
||||
!!! Note
|
||||
You can convert between data-formats using the [convert-data](#subcommand-convert-data) and [convert-trade-data](#subcommand-convert-trade-data) methods.
|
||||
You can convert between data-formats using the [convert-data](#sub-command-convert-data) and [convert-trade-data](#sub-command-convert-trade-data) methods.
|
||||
|
||||
#### Subcommand convert data
|
||||
#### Sub-command convert data
|
||||
|
||||
```
|
||||
usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
|
||||
[-d PATH] [--userdir PATH]
|
||||
[-p PAIRS [PAIRS ...]] --format-from
|
||||
{json,jsongz} --format-to {json,jsongz}
|
||||
[--erase]
|
||||
{json,jsongz,hdf5} --format-to
|
||||
{json,jsongz,hdf5} [--erase]
|
||||
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
|
||||
|
||||
optional arguments:
|
||||
@ -77,9 +107,9 @@ optional arguments:
|
||||
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
|
||||
Show profits for only these pairs. Pairs are space-
|
||||
separated.
|
||||
--format-from {json,jsongz}
|
||||
--format-from {json,jsongz,hdf5}
|
||||
Source format for data conversion.
|
||||
--format-to {json,jsongz}
|
||||
--format-to {json,jsongz,hdf5}
|
||||
Destination format for data conversion.
|
||||
--erase Clean all existing data for the selected
|
||||
exchange/pairs/timeframes.
|
||||
@ -94,9 +124,10 @@ Common arguments:
|
||||
details.
|
||||
-V, --version show program's version number and exit
|
||||
-c PATH, --config PATH
|
||||
Specify configuration file (default: `config.json`).
|
||||
Multiple --config options may be used. Can be set to
|
||||
`-` to read config from stdin.
|
||||
Specify configuration file (default:
|
||||
`userdir/config.json` or `config.json` whichever
|
||||
exists). Multiple --config options may be used. Can be
|
||||
set to `-` to read config from stdin.
|
||||
-d PATH, --datadir PATH
|
||||
Path to directory with historical backtesting data.
|
||||
--userdir PATH, --user-data-dir PATH
|
||||
@ -112,23 +143,23 @@ It'll also remove original json data files (`--erase` parameter).
|
||||
freqtrade convert-data --format-from json --format-to jsongz --datadir ~/.freqtrade/data/binance -t 5m 15m --erase
|
||||
```
|
||||
|
||||
#### Subcommand convert-trade data
|
||||
#### Sub-command convert trade data
|
||||
|
||||
```
|
||||
usage: freqtrade convert-trade-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
|
||||
[-d PATH] [--userdir PATH]
|
||||
[-p PAIRS [PAIRS ...]] --format-from
|
||||
{json,jsongz} --format-to {json,jsongz}
|
||||
[--erase]
|
||||
{json,jsongz,hdf5} --format-to
|
||||
{json,jsongz,hdf5} [--erase]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
|
||||
Show profits for only these pairs. Pairs are space-
|
||||
separated.
|
||||
--format-from {json,jsongz}
|
||||
--format-from {json,jsongz,hdf5}
|
||||
Source format for data conversion.
|
||||
--format-to {json,jsongz}
|
||||
--format-to {json,jsongz,hdf5}
|
||||
Destination format for data conversion.
|
||||
--erase Clean all existing data for the selected
|
||||
exchange/pairs/timeframes.
|
||||
@ -140,13 +171,15 @@ Common arguments:
|
||||
details.
|
||||
-V, --version show program's version number and exit
|
||||
-c PATH, --config PATH
|
||||
Specify configuration file (default: `config.json`).
|
||||
Multiple --config options may be used. Can be set to
|
||||
`-` to read config from stdin.
|
||||
Specify configuration file (default:
|
||||
`userdir/config.json` or `config.json` whichever
|
||||
exists). Multiple --config options may be used. Can be
|
||||
set to `-` to read config from stdin.
|
||||
-d PATH, --datadir PATH
|
||||
Path to directory with historical backtesting data.
|
||||
--userdir PATH, --user-data-dir PATH
|
||||
Path to userdata directory.
|
||||
|
||||
```
|
||||
|
||||
##### Example converting trades
|
||||
@ -158,21 +191,21 @@ It'll also remove original jsongz data files (`--erase` parameter).
|
||||
freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase
|
||||
```
|
||||
|
||||
### Subcommand list-data
|
||||
### Sub-command list-data
|
||||
|
||||
You can get a list of downloaded data using the `list-data` subcommand.
|
||||
You can get a list of downloaded data using the `list-data` sub-command.
|
||||
|
||||
```
|
||||
usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
|
||||
[--userdir PATH] [--exchange EXCHANGE]
|
||||
[--data-format-ohlcv {json,jsongz}]
|
||||
[--data-format-ohlcv {json,jsongz,hdf5}]
|
||||
[-p PAIRS [PAIRS ...]]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no
|
||||
config is provided.
|
||||
--data-format-ohlcv {json,jsongz}
|
||||
--data-format-ohlcv {json,jsongz,hdf5}
|
||||
Storage format for downloaded candle (OHLCV) data.
|
||||
(default: `json`).
|
||||
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
|
||||
@ -194,6 +227,7 @@ Common arguments:
|
||||
Path to directory with historical backtesting data.
|
||||
--userdir PATH, --user-data-dir PATH
|
||||
Path to userdata directory.
|
||||
|
||||
```
|
||||
|
||||
#### Example list-data
|
||||
@ -249,7 +283,7 @@ This will download historical candle (OHLCV) data for all the currency pairs you
|
||||
### Other Notes
|
||||
|
||||
- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`.
|
||||
- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust ratelimits etc.)
|
||||
- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.)
|
||||
- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`.
|
||||
- To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days).
|
||||
- Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data.
|
||||
@ -257,7 +291,7 @@ This will download historical candle (OHLCV) data for all the currency pairs you
|
||||
|
||||
### Trades (tick) data
|
||||
|
||||
By default, `download-data` subcommand downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
|
||||
By default, `download-data` sub-command downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
|
||||
This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes.
|
||||
|
||||
Since this data is large by default, the files use gzip by default. They are stored in your data-directory with the naming convention of `<pair>-trades.json.gz` (`ETH_BTC-trades.json.gz`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
|
||||
|
||||
@ -59,6 +59,7 @@ class HDF5DataHandler(IDataHandler):
|
||||
_data = data.copy()
|
||||
|
||||
filename = self._pair_data_filename(self._datadir, pair, timeframe)
|
||||
|
||||
ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc')
|
||||
ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date'])
|
||||
|
||||
@ -139,6 +140,7 @@ class HDF5DataHandler(IDataHandler):
|
||||
column sequence as in DEFAULT_TRADES_COLUMNS
|
||||
"""
|
||||
key = self._pair_trades_key(pair)
|
||||
|
||||
ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair),
|
||||
mode='a', complevel=9, complib='blosc')
|
||||
ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS),
|
||||
|
||||
@ -9,7 +9,8 @@ from pandas import DataFrame
|
||||
|
||||
from freqtrade.configuration import TimeRange
|
||||
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
|
||||
from freqtrade.data.converter import (ohlcv_to_dataframe,
|
||||
from freqtrade.data.converter import (clean_ohlcv_dataframe,
|
||||
ohlcv_to_dataframe,
|
||||
trades_remove_duplicates,
|
||||
trades_to_ohlcv)
|
||||
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
|
||||
@ -202,7 +203,10 @@ def _download_pair_history(datadir: Path,
|
||||
if data.empty:
|
||||
data = new_dataframe
|
||||
else:
|
||||
data = data.append(new_dataframe)
|
||||
# Run cleaning again to ensure there were no duplicate candles
|
||||
# Especially between existing and new data.
|
||||
data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair,
|
||||
fill_missing=False, drop_incomplete=False)
|
||||
|
||||
logger.debug("New Start: %s",
|
||||
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user