adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5456 from mablue/download_process

Browse Source 
Download-data log process added
This commit is contained in:
Matthias 2021-08-21 07:12:04 +02:00 committed by GitHub
commit 34456b9798
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 24 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
freqtrade/data/history/history_utils.py
View File

@ -117,10 +117,11 @@ def refresh_data(datadir: Path,
:param timerange: Limit data to be loaded to this timerange :param timerange: Limit data to be loaded to this timerange
""" """
data_handler = get_datahandler(datadir, data_format) data_handler = get_datahandler(datadir, data_format)
for pair in pairs: for idx, pair in enumerate(pairs):
_download_pair_history(pair=pair, timeframe=timeframe, process = f'{idx}/{len(pairs)}'
datadir=datadir, timerange=timerange, _download_pair_history(pair=pair, process=process,
exchange=exchange, data_handler=data_handler) timeframe=timeframe, datadir=datadir,
timerange=timerange, exchange=exchange, data_handler=data_handler)
def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
@ -153,13 +154,14 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
return data, start_ms return data, start_ms
def _download_pair_history(datadir: Path, def _download_pair_history(pair: str, *,
datadir: Path,
exchange: Exchange, exchange: Exchange,
pair: str, *,
new_pairs_days: int = 30,
timeframe: str = '5m', timeframe: str = '5m',
timerange: Optional[TimeRange] = None, process: str = '',
data_handler: IDataHandler = None) -> bool: new_pairs_days: int = 30,
data_handler: IDataHandler = None,
timerange: Optional[TimeRange] = None) -> bool:
""" """
Download latest candles from the exchange for the pair and timeframe passed in parameters Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that The data is downloaded starting from the last correct data that
@ -177,7 +179,7 @@ def _download_pair_history(datadir: Path,
try: try:
logger.info( logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} ' f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} '
f'and store in {datadir}.' f'and store in {datadir}.'
) )
@ -234,7 +236,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
""" """
pairs_not_available = [] pairs_not_available = []
data_handler = get_datahandler(datadir, data_format) data_handler = get_datahandler(datadir, data_format)
for pair in pairs: for idx, pair in enumerate(pairs, start=1):
if pair not in exchange.markets: if pair not in exchange.markets:
pairs_not_available.append(pair) pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...") logger.info(f"Skipping pair {pair}...")
@ -247,10 +249,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
f'Deleting existing data for pair {pair}, interval {timeframe}.') f'Deleting existing data for pair {pair}, interval {timeframe}.')
logger.info(f'Downloading pair {pair}, interval {timeframe}.') logger.info(f'Downloading pair {pair}, interval {timeframe}.')
_download_pair_history(datadir=datadir, exchange=exchange, process = f'{idx}/{len(pairs)}'
pair=pair, timeframe=str(timeframe), _download_pair_history(pair=pair, process=process,
new_pairs_days=new_pairs_days, datadir=datadir, exchange=exchange,
timerange=timerange, data_handler=data_handler) timerange=timerange, data_handler=data_handler,
timeframe=str(timeframe), new_pairs_days=new_pairs_days)
return pairs_not_available return pairs_not_available

10
tests/data/test_history.py
View File

@ -133,8 +133,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog,
load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC')
assert file.is_file() assert file.is_file()
assert log_has_re( assert log_has_re(
'Download history data for pair: "MEME/BTC", timeframe: 1m ' r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m '
'and store in .*', caplog r'and store in .*', caplog
) )
@ -278,8 +278,10 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
return_value=None) return_value=None)
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
exchange = get_patched_exchange(mocker, default_conf) exchange = get_patched_exchange(mocker, default_conf)
_download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') _download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC",
_download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') timeframe='1m')
_download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC",
timeframe='3m')
assert json_dump_mock.call_count == 2 assert json_dump_mock.call_count == 2