adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6753 from freqtrade/download_prepend

Browse Source 
Download prepend
This commit is contained in:
Matthias 2022-05-01 15:15:16 +02:00 committed by GitHub
commit 910addd02b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 109 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/data-download.md
View File

@ -30,6 +30,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[--data-format-ohlcv {json,jsongz,hdf5}]
[--data-format-trades {json,jsongz,hdf5}]
[--trading-mode {spot,margin,futures}]
[--prepend]
optional arguments:
-h, --help show this help message and exit
@ -62,6 +63,7 @@ optional arguments:
`jsongz`).
--trading-mode {spot,margin,futures}
Select Trading mode
--prepend Allow data prepending.
Common arguments:
-v, --verbose Verbose mode (-vv for more, -vvv to get all messages).
@ -157,10 +159,21 @@ freqtrade download-data --exchange binance --pairs .*/USDT
- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.)
- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`.
- To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days).
- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. Eventually set end dates are ignored.
- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020.
- Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data.
- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options.
#### Download additional data before the current timerange
Assuming you downloaded all data from 2022 (`--timerange 20220101-`) - but you'd now like to also backtest with earlier data.
You can do so by using the `--prepend` flag, combined with `--timerange` - specifying an end-date.
``` bash
freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT --prepend --timerange 20210101-20220101
```
!!! Note
Freqtrade will ignore the end-date in this mode if data is available, updating the end-date to the existing data start point.
### Data format

3
freqtrade/commands/arguments.py
View File

@ -72,7 +72,8 @@ ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive",
"timerange", "download_trades", "exchange", "timeframes",
"erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode"]
"erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode",
"prepend_data"]
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
"db_url", "trade_source", "export", "exportfilename",

5
freqtrade/commands/cli_options.py
View File

@ -443,6 +443,11 @@ AVAILABLE_CLI_OPTIONS = {
default=['1m', '5m'],
nargs='+',
),
"prepend_data": Arg(
'--prepend',
help='Allow data prepending.',
action='store_true',
),
"erase": Arg(
'--erase',
help='Clean all existing data for the selected exchange/pairs/timeframes.',

1
freqtrade/commands/data_commands.py
View File

@ -85,6 +85,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
trading_mode=config.get('trading_mode', 'spot'),
prepend=config.get('prepend_data', False)
)
except KeyboardInterrupt:

2
freqtrade/configuration/configuration.py
View File

@ -393,6 +393,8 @@ class Configuration:
self._args_to_config(config, argname='trade_source',
logstring='Using trades from: {}')
self._args_to_config(config, argname='prepend_data',
logstring='Prepend detected. Allowing data prepending.')
self._args_to_config(config, argname='erase',
logstring='Erase detected. Deleting existing data.')

50
freqtrade/data/history/history_utils.py
View File

@ -139,8 +139,9 @@ def _load_cached_data_for_updating(
timeframe: str,
timerange: Optional[TimeRange],
data_handler: IDataHandler,
candle_type: CandleType
) -> Tuple[DataFrame, Optional[int]]:
candle_type: CandleType,
prepend: bool = False,
) -> Tuple[DataFrame, Optional[int], Optional[int]]:
"""
Load cached data to download more data.
If timerange is passed in, checks whether data from an before the stored data will be
@ -150,9 +151,12 @@ def _load_cached_data_for_updating(
Note: Only used by download_pair_history().
"""
start = None
end = None
if timerange:
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
if timerange.stoptype == 'date':
end = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
# Intentionally don't pass timerange in - since we need to load the full dataset.
data = data_handler.ohlcv_load(pair, timeframe=timeframe,
@ -160,14 +164,17 @@ def _load_cached_data_for_updating(
drop_incomplete=True, warn_no_data=False,
candle_type=candle_type)
if not data.empty:
if start and start < data.iloc[0]['date']:
if not prepend and start and start < data.iloc[0]['date']:
# Earlier data than existing data requested, redownload all
data = DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS)
else:
start = data.iloc[-1]['date']
if prepend:
end = data.iloc[0]['date']
else:
start = data.iloc[-1]['date']
start_ms = int(start.timestamp() * 1000) if start else None
return data, start_ms
end_ms = int(end.timestamp() * 1000) if end else None
return data, start_ms, end_ms
def _download_pair_history(pair: str, *,
@ -180,6 +187,7 @@ def _download_pair_history(pair: str, *,
timerange: Optional[TimeRange] = None,
candle_type: CandleType,
erase: bool = False,
prepend: bool = False,
) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
@ -187,8 +195,6 @@ def _download_pair_history(pair: str, *,
exists in a cache. If timerange starts earlier than the data in the cache,
the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
@ -203,14 +209,17 @@ def _download_pair_history(pair: str, *,
if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type):
logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.')
logger.info(
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, '
f'candle type: {candle_type} and store in {datadir}.'
)
data, since_ms, until_ms = _load_cached_data_for_updating(
pair, timeframe, timerange,
data_handler=data_handler,
candle_type=candle_type,
prepend=prepend)
data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange,
data_handler=data_handler,
candle_type=candle_type)
logger.info(f'({process}) - Download history data for "{pair}", {timeframe}, '
f'{candle_type} and store in {datadir}.'
f'From {format_ms_time(since_ms) if since_ms else "start"} to '
f'{format_ms_time(until_ms) if until_ms else "now"}'
)
logger.debug("Current Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
@ -225,6 +234,7 @@ def _download_pair_history(pair: str, *,
days=-new_pairs_days).int_timestamp * 1000,
is_new_pair=data.empty,
candle_type=candle_type,
until_ms=until_ms if until_ms else None
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@ -257,6 +267,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
timerange: Optional[TimeRange] = None,
new_pairs_days: int = 30, erase: bool = False,
data_format: str = None,
prepend: bool = False,
) -> List[str]:
"""
Refresh stored ohlcv data for backtesting and hyperopt operations.
@ -280,7 +291,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
timerange=timerange, data_handler=data_handler,
timeframe=str(timeframe), new_pairs_days=new_pairs_days,
candle_type=candle_type,
erase=erase)
erase=erase, prepend=prepend)
if trading_mode == 'futures':
# Predefined candletype (and timeframe) depending on exchange
# Downloads what is necessary to backtest based on futures data.
@ -294,7 +305,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
timerange=timerange, data_handler=data_handler,
timeframe=str(tf_mark), new_pairs_days=new_pairs_days,
candle_type=funding_candle_type,
erase=erase)
erase=erase, prepend=prepend)
return pairs_not_available
@ -312,8 +323,9 @@ def _download_trades_history(exchange: Exchange,
try:
until = None
if (timerange and timerange.starttype == 'date'):
since = timerange.startts * 1000
if timerange:
if timerange.starttype == 'date':
since = timerange.startts * 1000
if timerange.stoptype == 'date':
until = timerange.stopts * 1000
else:

4
freqtrade/exchange/binance.py
View File

@ -95,6 +95,7 @@ class Binance(Exchange):
async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, candle_type: CandleType,
is_new_pair: bool = False, raise_: bool = False,
until_ms: int = None
) -> Tuple[str, str, str, List]:
"""
Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date
@ -115,7 +116,8 @@ class Binance(Exchange):
since_ms=since_ms,
is_new_pair=is_new_pair,
raise_=raise_,
candle_type=candle_type
candle_type=candle_type,
until_ms=until_ms,
)
def funding_fee_cutoff(self, open_date: datetime):

11
freqtrade/exchange/exchange.py
View File

@ -1645,7 +1645,8 @@ class Exchange:
def get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, candle_type: CandleType,
is_new_pair: bool = False) -> List:
is_new_pair: bool = False,
until_ms: int = None) -> List:
"""
Get candle history using asyncio and returns the list of candles.
Handles all async work for this.
@ -1653,13 +1654,14 @@ class Exchange:
:param pair: Pair to download
:param timeframe: Timeframe to get data for
:param since_ms: Timestamp in milliseconds to get history from
:param until_ms: Timestamp in milliseconds to get history up to
:param candle_type: '', mark, index, premiumIndex, or funding_rate
:return: List with candle (OHLCV) data
"""
pair, _, _, data = self.loop.run_until_complete(
self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe,
since_ms=since_ms, is_new_pair=is_new_pair,
candle_type=candle_type))
since_ms=since_ms, until_ms=until_ms,
is_new_pair=is_new_pair, candle_type=candle_type))
logger.info(f"Downloaded data for {pair} with length {len(data)}.")
return data
@ -1680,6 +1682,7 @@ class Exchange:
async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, candle_type: CandleType,
is_new_pair: bool = False, raise_: bool = False,
until_ms: int = None
) -> Tuple[str, str, str, List]:
"""
Download historic ohlcv
@ -1695,7 +1698,7 @@ class Exchange:
)
input_coroutines = [self._async_get_candle_history(
pair, timeframe, candle_type, since) for since in
range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)]
range(since_ms, until_ms or (arrow.utcnow().int_timestamp * 1000), one_call)]
data: List = []
# Chunk requests into batches of 100 to avoid overwelming ccxt Throttling

37
tests/data/test_history.py
View File

@ -149,8 +149,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog,
load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC', candle_type=candle_type)
assert file.is_file()
assert log_has_re(
r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m, '
r'candle type: spot and store in .*', caplog
r'\(0/1\) - Download history data for "MEME/BTC", 1m, '
r'spot and store in .*', caplog
)
@ -223,42 +223,65 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None:
# timeframe starts earlier than the cached data
# should fully update data
timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0)
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert data.empty
assert start_ts == test_data[0][0] - 1000
assert end_ts is None
# timeframe starts earlier than the cached data - prepending
timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0)
data, start_ts, end_ts = _load_cached_data_for_updating(
'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT, True)
assert_frame_equal(data, test_data_df.iloc[:-1])
assert start_ts == test_data[0][0] - 1000
assert end_ts == test_data[0][0]
# timeframe starts in the center of the cached data
# should return the cached data w/o the last item
timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0)
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert_frame_equal(data, test_data_df.iloc[:-1])
assert test_data[-2][0] <= start_ts < test_data[-1][0]
assert end_ts is None
# timeframe starts after the cached data
# should return the cached data w/o the last item
timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 100, 0)
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert_frame_equal(data, test_data_df.iloc[:-1])
assert test_data[-2][0] <= start_ts < test_data[-1][0]
assert end_ts is None
# no datafile exist
# should return timestamp start time
timerange = TimeRange('date', None, now_ts - 10000, 0)
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'NONEXIST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert data.empty
assert start_ts == (now_ts - 10000) * 1000
assert end_ts is None
# no datafile exist
# should return timestamp start and end time time
timerange = TimeRange('date', 'date', now_ts - 1000000, now_ts - 100000)
data, start_ts, end_ts = _load_cached_data_for_updating(
'NONEXIST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert data.empty
assert start_ts == (now_ts - 1000000) * 1000
assert end_ts == (now_ts - 100000) * 1000
# no datafile exist, no timeframe is set
# should return an empty array and None
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'NONEXIST/BTC', '1m', None, data_handler, CandleType.SPOT)
assert data.empty
assert start_ts is None
assert end_ts is None
@pytest.mark.parametrize('candle_type,subdir,file_tail', [

14
tests/exchange/test_exchange.py
View File

@ -1983,6 +1983,20 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_
assert exchange._api_async.fetch_ohlcv.call_count > 200
assert res[0] == ohlcv[0]
exchange._api_async.fetch_ohlcv.reset_mock()
end_ts = 1_500_500_000_000
start_ts = 1_500_000_000_000
respair, restf, _, res = await exchange._async_get_historic_ohlcv(
pair, "5m", since_ms=start_ts, candle_type=candle_type, is_new_pair=False,
until_ms=end_ts
)
# Required candles
candles = (end_ts - start_ts) / 300_000
exp = candles // exchange.ohlcv_candle_limit('5m') + 1
# Depending on the exchange, this should be called between 1 and 6 times.
assert exchange._api_async.fetch_ohlcv.call_count == exp
@pytest.mark.parametrize('candle_type', [CandleType.FUTURES, CandleType.MARK, CandleType.SPOT])
def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None: