diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index e0f9f67db..2dacce8c6 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -122,21 +122,34 @@ def load_data(datadir: Optional[Path], refresh_pairs: bool = False, exchange: Optional[Exchange] = None, timerange: TimeRange = TimeRange(None, None, 0, 0), - fill_up_missing: bool = True) -> Dict[str, DataFrame]: + fill_up_missing: bool = True, + live: bool = False + ) -> Dict[str, DataFrame]: """ Loads ticker history data for a list of pairs the given parameters :return: dict(:) """ - result = {} + result: Dict[str, DataFrame] = {} + if live: + if exchange: + logger.info('Live: Downloading data for all defined pairs ...') + exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) + result = {key[0]: value for key, value in exchange._klines.items() if value is not None} + else: + raise OperationalException( + "Exchange needs to be initialized when using live data." + ) + else: + logger.info('Using local backtesting data ...') - for pair in pairs: - hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, - datadir=datadir, timerange=timerange, - refresh_pairs=refresh_pairs, - exchange=exchange, - fill_up_missing=fill_up_missing) - if hist is not None: - result[pair] = hist + for pair in pairs: + hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, + datadir=datadir, timerange=timerange, + refresh_pairs=refresh_pairs, + exchange=exchange, + fill_up_missing=fill_up_missing) + if hist is not None: + result[pair] = hist return result diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index bdd42943b..76c6556fa 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -401,24 +401,17 @@ class Backtesting(object): logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - if self.config.get('live'): - logger.info('Downloading data for all pairs in whitelist ...') - self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs]) - data = {key[0]: value for key, value in self.exchange._klines.items()} - - else: - logger.info('Using local backtesting data (using whitelist in given config) ...') - - timerange = Arguments.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - data = history.load_data( - datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, - pairs=pairs, - ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.exchange, - timerange=timerange - ) + timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + data = history.load_data( + datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + exchange=self.exchange, + timerange=timerange, + live=self.config.get('live', False) + ) if not data: logger.critical("No data found. Terminating.") diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 4d70d4cdd..a13bc34af 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -5,6 +5,7 @@ import os import uuid from pathlib import Path from shutil import copyfile +from unittest.mock import MagicMock import arrow import pytest @@ -20,7 +21,8 @@ from freqtrade.data.history import (download_pair_history, from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.strategy.default_strategy import DefaultStrategy -from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange +from freqtrade.tests.conftest import (get_patched_exchange, log_has, + patch_exchange) # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -136,6 +138,31 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau _clean_test_file(file) +def test_load_data_live(default_conf, mocker, caplog) -> None: + refresh_mock = MagicMock() + mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) + exchange = get_patched_exchange(mocker, default_conf) + + history.load_data(datadir=None, ticker_interval='5m', + pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], + live=True, + exchange=exchange) + assert refresh_mock.call_count == 1 + assert len(refresh_mock.call_args_list[0][0][0]) == 2 + assert log_has('Live: Downloading data for all defined pairs ...', caplog.record_tuples) + + +def test_load_data_live_noexchange(default_conf, mocker, caplog) -> None: + + with pytest.raises(OperationalException, + match=r'Exchange needs to be initialized when using live data.'): + history.load_data(datadir=None, ticker_interval='5m', + pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], + exchange=None, + live=True, + ) + + def test_testdata_path() -> None: assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 5b42cae34..3f88a8d6c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -105,7 +105,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None: def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, - timerange=None, exchange=None): + timerange=None, exchange=None, live=False): tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', fill_missing=True)} return pairdata @@ -492,7 +492,6 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: backtesting.start() # check the logs, that will contain the backtest result exists = [ - 'Using local backtesting data (using whitelist in given config) ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' @@ -857,7 +856,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Using data folder: freqtrade/tests/testdata ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Downloading data for all pairs in whitelist ...', + 'Live: Downloading data for all defined pairs ...', 'Backtesting with data from 2017-11-14T19:31:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...' @@ -916,7 +915,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): 'Using data folder: freqtrade/tests/testdata ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Downloading data for all pairs in whitelist ...', + 'Live: Downloading data for all defined pairs ...', 'Backtesting with data from 2017-11-14T19:31:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ba549deb5..27932b559 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -139,21 +139,15 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args): ticker_interval = strategy.ticker_interval timerange = Arguments.parse_timerange(args.timerange) - tickers = {} - if args.live: - logger.info('Downloading pairs.') - exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) - for pair in pairs: - tickers[pair] = exchange.klines((pair, ticker_interval)) - else: - tickers = history.load_data( - datadir=Path(str(_CONF.get("datadir"))), - pairs=pairs, - ticker_interval=ticker_interval, - refresh_pairs=_CONF.get('refresh_pairs', False), - timerange=timerange, - exchange=Exchange(_CONF) - ) + tickers = history.load_data( + datadir=Path(str(_CONF.get("datadir"))), + pairs=pairs, + ticker_interval=ticker_interval, + refresh_pairs=_CONF.get('refresh_pairs', False), + timerange=timerange, + exchange=Exchange(_CONF), + live=args.live, + ) # No ticker found, impossible to download, len mismatch for pair, data in tickers.copy().items():