From e2b83624a397b85807d23f7b5353ade9727e54a3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 17 May 2019 19:05:36 +0300 Subject: [PATCH] data/history cleanup --- freqtrade/data/history.py | 38 ++++++++++++++++------------ freqtrade/tests/data/test_history.py | 29 ++++++++++++++++----- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 4dba1b760..86d3c3071 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -90,13 +90,8 @@ def load_pair_history(pair: str, :return: DataFrame with ohlcv data """ - # If the user force the refresh of pairs + # The user forced the refresh of pairs if refresh_pairs: - if not exchange: - raise OperationalException("Exchange needs to be initialized when " - "calling load_data with refresh_pairs=True") - - logger.info('Download data for pair and store them in %s', datadir) download_pair_history(datadir=datadir, exchange=exchange, pair=pair, @@ -115,10 +110,11 @@ def load_pair_history(pair: str, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing) else: - logger.warning('No data for pair: "%s", Interval: %s. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' - 'script to download the data', - pair, ticker_interval) + logger.warning( + f'No history data for pair: "{pair}", interval: {ticker_interval}. ' + 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'script to download the data' + ) return None @@ -190,7 +186,7 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str, def download_pair_history(datadir: Optional[Path], - exchange: Exchange, + exchange: Optional[Exchange], pair: str, ticker_interval: str = '5m', timerange: Optional[TimeRange] = None) -> bool: @@ -201,18 +197,26 @@ def download_pair_history(datadir: Optional[Path], the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data + :param pair: pair to download :param ticker_interval: ticker interval :param timerange: range of time to download :return: bool with success state - """ + if not exchange: + raise OperationalException( + "Exchange needs to be initialized when downloading pair history data" + ) + try: path = make_testdata_path(datadir) filepair = pair.replace("/", "_") filename = path.joinpath(f'{filepair}-{ticker_interval}.json') - logger.info('Download the pair: "%s", Interval: %s', pair, ticker_interval) + logger.info( + f'Download history data for pair: "{pair}", interval: {ticker_interval} ' + f'and store in {datadir}.' + ) data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) @@ -231,7 +235,9 @@ def download_pair_history(datadir: Optional[Path], misc.file_dump_json(filename, data) return True - except BaseException: - logger.info('Failed to download the pair: "%s", Interval: %s', - pair, ticker_interval) + + except Exception: + logger.error( + f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}.' + ) return False diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 14ec99042..15442f577 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -59,7 +59,11 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(mocker, caplog, default_conf) -> None: ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=None) assert isinstance(ld, DataFrame) - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples) + assert not log_has( + 'Download history data for pair: "UNITTEST/BTC", interval: 30m ' + 'and store in None.', + caplog.record_tuples + ) def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: @@ -67,7 +71,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: assert not isinstance(ld, DataFrame) assert ld is None assert log_has( - 'No data for pair: "UNITTEST/BTC", Interval: 7m. ' + 'No history data for pair: "UNITTEST/BTC", interval: 7m. ' 'Use --refresh-pairs-cached option or download_backtest_data.py ' 'script to download the data', caplog.record_tuples @@ -80,7 +84,11 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: _backup_file(file, copy_file=True) history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC']) assert os.path.isfile(file) is True - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 1m', caplog.record_tuples) + assert not log_has( + 'Download history data for pair: "UNITTEST/BTC", interval: 1m ' + 'and store in None.', + caplog.record_tuples + ) _clean_test_file(file) @@ -100,7 +108,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau pair='MEME/BTC') assert os.path.isfile(file) is False assert log_has( - 'No data for pair: "MEME/BTC", Interval: 1m. ' + 'No history data for pair: "MEME/BTC", interval: 1m. ' 'Use --refresh-pairs-cached option or download_backtest_data.py ' 'script to download the data', caplog.record_tuples @@ -113,7 +121,11 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau exchange=exchange, pair='MEME/BTC') assert os.path.isfile(file) is True - assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + assert log_has( + 'Download history data for pair: "MEME/BTC", interval: 1m ' + 'and store in None.', + caplog.record_tuples + ) with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): history.load_pair_history(datadir=None, ticker_interval='1m', @@ -293,7 +305,7 @@ def test_download_pair_history2(mocker, default_conf) -> None: def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf) -> None: mocker.patch('freqtrade.exchange.Exchange.get_history', - side_effect=BaseException('File Error')) + side_effect=Exception('File Error')) exchange = get_patched_exchange(mocker, default_conf) @@ -308,7 +320,10 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) - assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + assert log_has( + 'Failed to download history data for pair: "MEME/BTC", interval: 1m.', + caplog.record_tuples + ) def test_load_tickerdata_file() -> None: