adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5456 from mablue/download_process

Browse Source 
Download-data log process added
This commit is contained in:
Matthias 2021-08-21 07:12:04 +02:00 committed by GitHub
commit 34456b9798
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 24 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
freqtrade/data/history/history_utils.py
View File

@ -117,10 +117,11 @@ def refresh_data(datadir: Path,
:param timerange: Limit data to be loaded to this timerange
"""
data_handler = get_datahandler(datadir, data_format)
for pair in pairs:
_download_pair_history(pair=pair, timeframe=timeframe,
datadir=datadir, timerange=timerange,
exchange=exchange, data_handler=data_handler)
for idx, pair in enumerate(pairs):
process = f'{idx}/{len(pairs)}'
_download_pair_history(pair=pair, process=process,
timeframe=timeframe, datadir=datadir,
timerange=timerange, exchange=exchange, data_handler=data_handler)
def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
@ -153,13 +154,14 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
return data, start_ms
def _download_pair_history(datadir: Path,
def _download_pair_history(pair: str, *,
datadir: Path,
exchange: Exchange,
pair: str, *,
new_pairs_days: int = 30,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler = None) -> bool:
process: str = '',
new_pairs_days: int = 30,
data_handler: IDataHandler = None,
timerange: Optional[TimeRange] = None) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
@ -177,7 +179,7 @@ def _download_pair_history(datadir: Path,
try:
logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} '
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} '
f'and store in {datadir}.'
)
@ -234,7 +236,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
"""
pairs_not_available = []
data_handler = get_datahandler(datadir, data_format)
for pair in pairs:
for idx, pair in enumerate(pairs, start=1):
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
@ -247,10 +249,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
f'Deleting existing data for pair {pair}, interval {timeframe}.')
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
_download_pair_history(datadir=datadir, exchange=exchange,
pair=pair, timeframe=str(timeframe),
new_pairs_days=new_pairs_days,
timerange=timerange, data_handler=data_handler)
process = f'{idx}/{len(pairs)}'
_download_pair_history(pair=pair, process=process,
datadir=datadir, exchange=exchange,
timerange=timerange, data_handler=data_handler,
timeframe=str(timeframe), new_pairs_days=new_pairs_days)
return pairs_not_available

10
tests/data/test_history.py
View File

@ -133,8 +133,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog,
load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC')
assert file.is_file()
assert log_has_re(
'Download history data for pair: "MEME/BTC", timeframe: 1m '
'and store in .*', caplog
r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m '
r'and store in .*', caplog
)
@ -278,8 +278,10 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
return_value=None)
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
exchange = get_patched_exchange(mocker, default_conf)
_download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m')
_download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m')
_download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC",
timeframe='1m')
_download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC",
timeframe='3m')
assert json_dump_mock.call_count == 2