From c2f6897d8b722a1c92cd845f63adfe1270c7bff7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 May 2019 20:10:48 +0200 Subject: [PATCH 1/2] Move download of live data to load_data Avoids code duplication in backtesting and plot_dataframe --- freqtrade/data/history.py | 26 +++++++++++++++++--------- freqtrade/optimize/backtesting.py | 29 +++++++++++------------------ 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index e0f9f67db..29a7f3478 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -122,21 +122,29 @@ def load_data(datadir: Optional[Path], refresh_pairs: bool = False, exchange: Optional[Exchange] = None, timerange: TimeRange = TimeRange(None, None, 0, 0), - fill_up_missing: bool = True) -> Dict[str, DataFrame]: + fill_up_missing: bool = True, + live: bool = False + ) -> Dict[str, DataFrame]: """ Loads ticker history data for a list of pairs the given parameters :return: dict(:) """ result = {} + if live: + logger.info('Live: Downloading data for all defined pairs ...') + exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) + result = {key[0]: value for key, value in exchange._klines.items() if value is not None} + else: + logger.info('Using local backtesting data ...') - for pair in pairs: - hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, - datadir=datadir, timerange=timerange, - refresh_pairs=refresh_pairs, - exchange=exchange, - fill_up_missing=fill_up_missing) - if hist is not None: - result[pair] = hist + for pair in pairs: + hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, + datadir=datadir, timerange=timerange, + refresh_pairs=refresh_pairs, + exchange=exchange, + fill_up_missing=fill_up_missing) + if hist is not None: + result[pair] = hist return result diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index bdd42943b..76c6556fa 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -401,24 +401,17 @@ class Backtesting(object): logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - if self.config.get('live'): - logger.info('Downloading data for all pairs in whitelist ...') - self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs]) - data = {key[0]: value for key, value in self.exchange._klines.items()} - - else: - logger.info('Using local backtesting data (using whitelist in given config) ...') - - timerange = Arguments.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - data = history.load_data( - datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, - pairs=pairs, - ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.exchange, - timerange=timerange - ) + timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + data = history.load_data( + datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + exchange=self.exchange, + timerange=timerange, + live=self.config.get('live', False) + ) if not data: logger.critical("No data found. Terminating.") From 15984b5c432b0f19fc1a8ace8ab6f36427b16e02 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 May 2019 20:25:07 +0200 Subject: [PATCH 2/2] Adjust some tests - implement new "live" method to plot_script --- freqtrade/data/history.py | 13 ++++++--- freqtrade/tests/data/test_history.py | 29 +++++++++++++++++++- freqtrade/tests/optimize/test_backtesting.py | 7 ++--- scripts/plot_dataframe.py | 24 ++++++---------- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 29a7f3478..2dacce8c6 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -129,11 +129,16 @@ def load_data(datadir: Optional[Path], Loads ticker history data for a list of pairs the given parameters :return: dict(:) """ - result = {} + result: Dict[str, DataFrame] = {} if live: - logger.info('Live: Downloading data for all defined pairs ...') - exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) - result = {key[0]: value for key, value in exchange._klines.items() if value is not None} + if exchange: + logger.info('Live: Downloading data for all defined pairs ...') + exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) + result = {key[0]: value for key, value in exchange._klines.items() if value is not None} + else: + raise OperationalException( + "Exchange needs to be initialized when using live data." + ) else: logger.info('Using local backtesting data ...') diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 4d70d4cdd..a13bc34af 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -5,6 +5,7 @@ import os import uuid from pathlib import Path from shutil import copyfile +from unittest.mock import MagicMock import arrow import pytest @@ -20,7 +21,8 @@ from freqtrade.data.history import (download_pair_history, from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.strategy.default_strategy import DefaultStrategy -from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange +from freqtrade.tests.conftest import (get_patched_exchange, log_has, + patch_exchange) # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -136,6 +138,31 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau _clean_test_file(file) +def test_load_data_live(default_conf, mocker, caplog) -> None: + refresh_mock = MagicMock() + mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) + exchange = get_patched_exchange(mocker, default_conf) + + history.load_data(datadir=None, ticker_interval='5m', + pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], + live=True, + exchange=exchange) + assert refresh_mock.call_count == 1 + assert len(refresh_mock.call_args_list[0][0][0]) == 2 + assert log_has('Live: Downloading data for all defined pairs ...', caplog.record_tuples) + + +def test_load_data_live_noexchange(default_conf, mocker, caplog) -> None: + + with pytest.raises(OperationalException, + match=r'Exchange needs to be initialized when using live data.'): + history.load_data(datadir=None, ticker_interval='5m', + pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], + exchange=None, + live=True, + ) + + def test_testdata_path() -> None: assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 5b42cae34..3f88a8d6c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -105,7 +105,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None: def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, - timerange=None, exchange=None): + timerange=None, exchange=None, live=False): tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', fill_missing=True)} return pairdata @@ -492,7 +492,6 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: backtesting.start() # check the logs, that will contain the backtest result exists = [ - 'Using local backtesting data (using whitelist in given config) ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' @@ -857,7 +856,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Using data folder: freqtrade/tests/testdata ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Downloading data for all pairs in whitelist ...', + 'Live: Downloading data for all defined pairs ...', 'Backtesting with data from 2017-11-14T19:31:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...' @@ -916,7 +915,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): 'Using data folder: freqtrade/tests/testdata ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Downloading data for all pairs in whitelist ...', + 'Live: Downloading data for all defined pairs ...', 'Backtesting with data from 2017-11-14T19:31:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ba549deb5..27932b559 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -139,21 +139,15 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args): ticker_interval = strategy.ticker_interval timerange = Arguments.parse_timerange(args.timerange) - tickers = {} - if args.live: - logger.info('Downloading pairs.') - exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) - for pair in pairs: - tickers[pair] = exchange.klines((pair, ticker_interval)) - else: - tickers = history.load_data( - datadir=Path(str(_CONF.get("datadir"))), - pairs=pairs, - ticker_interval=ticker_interval, - refresh_pairs=_CONF.get('refresh_pairs', False), - timerange=timerange, - exchange=Exchange(_CONF) - ) + tickers = history.load_data( + datadir=Path(str(_CONF.get("datadir"))), + pairs=pairs, + ticker_interval=ticker_interval, + refresh_pairs=_CONF.get('refresh_pairs', False), + timerange=timerange, + exchange=Exchange(_CONF), + live=args.live, + ) # No ticker found, impossible to download, len mismatch for pair, data in tickers.copy().items():