adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Download-data log process added

Browse Source 
Download-data log process added

pytest assert solved
This commit is contained in:
Masoud Azizi 2021-08-19 18:34:02 +00:00
parent 695a1e21bf
commit 40ad451019
2 changed files with 24 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
freqtrade/data/history/history_utils.py
View File

@ -117,10 +117,11 @@ def refresh_data(datadir: Path,
:param timerange: Limit data to be loaded to this timerange :param timerange: Limit data to be loaded to this timerange
""" """
data_handler = get_datahandler(datadir, data_format) data_handler = get_datahandler(datadir, data_format)
for pair in pairs: for idx, pair in enumerate(pairs):
_download_pair_history(pair=pair, timeframe=timeframe, process = f'{idx}/{len(pairs)}'
datadir=datadir, timerange=timerange, _download_pair_history(pair=pair, process=process,
exchange=exchange, data_handler=data_handler) timeframe=timeframe, datadir=datadir,
timerange=timerange, exchange=exchange, data_handler=data_handler)
def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
@ -153,13 +154,14 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
return data, start_ms return data, start_ms
def _download_pair_history(datadir: Path, def _download_pair_history(pair: str, *,
datadir: Path,
exchange: Exchange, exchange: Exchange,
pair: str, *,
new_pairs_days: int = 30,
timeframe: str = '5m', timeframe: str = '5m',
timerange: Optional[TimeRange] = None, process: str = '',
data_handler: IDataHandler = None) -> bool: new_pairs_days: int = 30,
data_handler: IDataHandler = None,
timerange: Optional[TimeRange] = None) -> bool:
""" """
Download latest candles from the exchange for the pair and timeframe passed in parameters Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that The data is downloaded starting from the last correct data that
@ -177,7 +179,7 @@ def _download_pair_history(datadir: Path,
try: try:
logger.info( logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} ' f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} '
f'and store in {datadir}.' f'and store in {datadir}.'
) )
@ -234,7 +236,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
""" """
pairs_not_available = [] pairs_not_available = []
data_handler = get_datahandler(datadir, data_format) data_handler = get_datahandler(datadir, data_format)
for pair in pairs: for idx, pair in enumerate(pairs, start=1):
if pair not in exchange.markets: if pair not in exchange.markets:
pairs_not_available.append(pair) pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...") logger.info(f"Skipping pair {pair}...")
@ -247,10 +249,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
f'Deleting existing data for pair {pair}, interval {timeframe}.') f'Deleting existing data for pair {pair}, interval {timeframe}.')
logger.info(f'Downloading pair {pair}, interval {timeframe}.') logger.info(f'Downloading pair {pair}, interval {timeframe}.')
_download_pair_history(datadir=datadir, exchange=exchange, process = f'{idx}/{len(pairs)}'
pair=pair, timeframe=str(timeframe), _download_pair_history(pair=pair, process=process,
new_pairs_days=new_pairs_days, datadir=datadir, exchange=exchange,
timerange=timerange, data_handler=data_handler) timerange=timerange, data_handler=data_handler,
timeframe=str(timeframe), new_pairs_days=new_pairs_days)
return pairs_not_available return pairs_not_available

10
tests/data/test_history.py
View File

@ -133,8 +133,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog,
load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC')
assert file.is_file() assert file.is_file()
assert log_has_re( assert log_has_re(
'Download history data for pair: "MEME/BTC", timeframe: 1m ' 'Download history data for pair: "MEME/BTC" (0/1), timeframe: 1m '
'and store in .*', caplog 'and store in', caplog
) )
@ -278,8 +278,10 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
return_value=None) return_value=None)
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
exchange = get_patched_exchange(mocker, default_conf) exchange = get_patched_exchange(mocker, default_conf)
_download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') _download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC",
_download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') timeframe='1m')
_download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC",
timeframe='3m')
assert json_dump_mock.call_count == 2 assert json_dump_mock.call_count == 2