From 40ad4510194e3eba6512fa1f2ca7d071ae38babd Mon Sep 17 00:00:00 2001 From: Masoud Azizi Date: Thu, 19 Aug 2021 18:34:02 +0000 Subject: [PATCH 1/2] Download-data log process added Download-data log process added pytest assert solved --- freqtrade/data/history/history_utils.py | 33 ++++++++++++++----------- tests/data/test_history.py | 10 +++++--- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 1459dfd78..6f125aaa9 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -117,10 +117,11 @@ def refresh_data(datadir: Path, :param timerange: Limit data to be loaded to this timerange """ data_handler = get_datahandler(datadir, data_format) - for pair in pairs: - _download_pair_history(pair=pair, timeframe=timeframe, - datadir=datadir, timerange=timerange, - exchange=exchange, data_handler=data_handler) + for idx, pair in enumerate(pairs): + process = f'{idx}/{len(pairs)}' + _download_pair_history(pair=pair, process=process, + timeframe=timeframe, datadir=datadir, + timerange=timerange, exchange=exchange, data_handler=data_handler) def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], @@ -153,13 +154,14 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona return data, start_ms -def _download_pair_history(datadir: Path, +def _download_pair_history(pair: str, *, + datadir: Path, exchange: Exchange, - pair: str, *, - new_pairs_days: int = 30, timeframe: str = '5m', - timerange: Optional[TimeRange] = None, - data_handler: IDataHandler = None) -> bool: + process: str = '', + new_pairs_days: int = 30, + data_handler: IDataHandler = None, + timerange: Optional[TimeRange] = None) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that @@ -177,7 +179,7 @@ def _download_pair_history(datadir: Path, try: logger.info( - f'Download history data for pair: "{pair}", timeframe: {timeframe} ' + f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} ' f'and store in {datadir}.' ) @@ -234,7 +236,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes """ pairs_not_available = [] data_handler = get_datahandler(datadir, data_format) - for pair in pairs: + for idx, pair in enumerate(pairs, start=1): if pair not in exchange.markets: pairs_not_available.append(pair) logger.info(f"Skipping pair {pair}...") @@ -247,10 +249,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes f'Deleting existing data for pair {pair}, interval {timeframe}.') logger.info(f'Downloading pair {pair}, interval {timeframe}.') - _download_pair_history(datadir=datadir, exchange=exchange, - pair=pair, timeframe=str(timeframe), - new_pairs_days=new_pairs_days, - timerange=timerange, data_handler=data_handler) + process = f'{idx}/{len(pairs)}' + _download_pair_history(pair=pair, process=process, + datadir=datadir, exchange=exchange, + timerange=timerange, data_handler=data_handler, + timeframe=str(timeframe), new_pairs_days=new_pairs_days) return pairs_not_available diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 13d22ebb7..bcc2bf052 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -133,8 +133,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') assert file.is_file() assert log_has_re( - 'Download history data for pair: "MEME/BTC", timeframe: 1m ' - 'and store in .*', caplog + 'Download history data for pair: "MEME/BTC" (0/1), timeframe: 1m ' + 'and store in', caplog ) @@ -278,8 +278,10 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) - _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') - _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') + _download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC", + timeframe='1m') + _download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC", + timeframe='3m') assert json_dump_mock.call_count == 2 From 127f470bc3928b052fc5ac00d85f4422ae4e5f1f Mon Sep 17 00:00:00 2001 From: Masoud Azizi Date: Fri, 20 Aug 2021 20:28:10 +0000 Subject: [PATCH 2/2] .* ADDED --- tests/data/test_history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index bcc2bf052..e9d2c3638 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -133,8 +133,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') assert file.is_file() assert log_has_re( - 'Download history data for pair: "MEME/BTC" (0/1), timeframe: 1m ' - 'and store in', caplog + r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m ' + r'and store in .*', caplog )