Merge pull request #1892 from freqtrade/ref/live_data

refactor `--live` handling
This commit is contained in:
Matthias 2019-06-01 06:20:11 +02:00 committed by GitHub
commit f04089ef1e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 74 additions and 48 deletions

View File

@ -122,12 +122,25 @@ def load_data(datadir: Optional[Path],
refresh_pairs: bool = False, refresh_pairs: bool = False,
exchange: Optional[Exchange] = None, exchange: Optional[Exchange] = None,
timerange: TimeRange = TimeRange(None, None, 0, 0), timerange: TimeRange = TimeRange(None, None, 0, 0),
fill_up_missing: bool = True) -> Dict[str, DataFrame]: fill_up_missing: bool = True,
live: bool = False
) -> Dict[str, DataFrame]:
""" """
Loads ticker history data for a list of pairs the given parameters Loads ticker history data for a list of pairs the given parameters
:return: dict(<pair>:<tickerlist>) :return: dict(<pair>:<tickerlist>)
""" """
result = {} result: Dict[str, DataFrame] = {}
if live:
if exchange:
logger.info('Live: Downloading data for all defined pairs ...')
exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs])
result = {key[0]: value for key, value in exchange._klines.items() if value is not None}
else:
raise OperationalException(
"Exchange needs to be initialized when using live data."
)
else:
logger.info('Using local backtesting data ...')
for pair in pairs: for pair in pairs:
hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, hist = load_pair_history(pair=pair, ticker_interval=ticker_interval,

View File

@ -401,14 +401,6 @@ class Backtesting(object):
logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
logger.info('Using stake_amount: %s ...', self.config['stake_amount']) logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
if self.config.get('live'):
logger.info('Downloading data for all pairs in whitelist ...')
self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs])
data = {key[0]: value for key, value in self.exchange._klines.items()}
else:
logger.info('Using local backtesting data (using whitelist in given config) ...')
timerange = Arguments.parse_timerange(None if self.config.get( timerange = Arguments.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange'))) 'timerange') is None else str(self.config.get('timerange')))
data = history.load_data( data = history.load_data(
@ -417,7 +409,8 @@ class Backtesting(object):
ticker_interval=self.ticker_interval, ticker_interval=self.ticker_interval,
refresh_pairs=self.config.get('refresh_pairs', False), refresh_pairs=self.config.get('refresh_pairs', False),
exchange=self.exchange, exchange=self.exchange,
timerange=timerange timerange=timerange,
live=self.config.get('live', False)
) )
if not data: if not data:

View File

@ -5,6 +5,7 @@ import os
import uuid import uuid
from pathlib import Path from pathlib import Path
from shutil import copyfile from shutil import copyfile
from unittest.mock import MagicMock
import arrow import arrow
import pytest import pytest
@ -20,7 +21,8 @@ from freqtrade.data.history import (download_pair_history,
from freqtrade.exchange import timeframe_to_minutes from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import file_dump_json from freqtrade.misc import file_dump_json
from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.default_strategy import DefaultStrategy
from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange from freqtrade.tests.conftest import (get_patched_exchange, log_has,
patch_exchange)
# Change this if modifying UNITTEST/BTC testdatafile # Change this if modifying UNITTEST/BTC testdatafile
_BTC_UNITTEST_LENGTH = 13681 _BTC_UNITTEST_LENGTH = 13681
@ -136,6 +138,31 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau
_clean_test_file(file) _clean_test_file(file)
def test_load_data_live(default_conf, mocker, caplog) -> None:
refresh_mock = MagicMock()
mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)
exchange = get_patched_exchange(mocker, default_conf)
history.load_data(datadir=None, ticker_interval='5m',
pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
live=True,
exchange=exchange)
assert refresh_mock.call_count == 1
assert len(refresh_mock.call_args_list[0][0][0]) == 2
assert log_has('Live: Downloading data for all defined pairs ...', caplog.record_tuples)
def test_load_data_live_noexchange(default_conf, mocker, caplog) -> None:
with pytest.raises(OperationalException,
match=r'Exchange needs to be initialized when using live data.'):
history.load_data(datadir=None, ticker_interval='5m',
pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
exchange=None,
live=True,
)
def test_testdata_path() -> None: def test_testdata_path() -> None:
assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None))

View File

@ -105,7 +105,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None:
def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
timerange=None, exchange=None): timerange=None, exchange=None, live=False):
tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', fill_missing=True)} pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', fill_missing=True)}
return pairdata return pairdata
@ -492,7 +492,6 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
backtesting.start() backtesting.start()
# check the logs, that will contain the backtest result # check the logs, that will contain the backtest result
exists = [ exists = [
'Using local backtesting data (using whitelist in given config) ...',
'Using stake_currency: BTC ...', 'Using stake_currency: BTC ...',
'Using stake_amount: 0.001 ...', 'Using stake_amount: 0.001 ...',
'Backtesting with data from 2017-11-14T21:17:00+00:00 ' 'Backtesting with data from 2017-11-14T21:17:00+00:00 '
@ -857,7 +856,7 @@ def test_backtest_start_live(default_conf, mocker, caplog):
'Using data folder: freqtrade/tests/testdata ...', 'Using data folder: freqtrade/tests/testdata ...',
'Using stake_currency: BTC ...', 'Using stake_currency: BTC ...',
'Using stake_amount: 0.001 ...', 'Using stake_amount: 0.001 ...',
'Downloading data for all pairs in whitelist ...', 'Live: Downloading data for all defined pairs ...',
'Backtesting with data from 2017-11-14T19:31:00+00:00 ' 'Backtesting with data from 2017-11-14T19:31:00+00:00 '
'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'up to 2017-11-14T22:58:00+00:00 (0 days)..',
'Parameter --enable-position-stacking detected ...' 'Parameter --enable-position-stacking detected ...'
@ -916,7 +915,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog):
'Using data folder: freqtrade/tests/testdata ...', 'Using data folder: freqtrade/tests/testdata ...',
'Using stake_currency: BTC ...', 'Using stake_currency: BTC ...',
'Using stake_amount: 0.001 ...', 'Using stake_amount: 0.001 ...',
'Downloading data for all pairs in whitelist ...', 'Live: Downloading data for all defined pairs ...',
'Backtesting with data from 2017-11-14T19:31:00+00:00 ' 'Backtesting with data from 2017-11-14T19:31:00+00:00 '
'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'up to 2017-11-14T22:58:00+00:00 (0 days)..',
'Parameter --enable-position-stacking detected ...', 'Parameter --enable-position-stacking detected ...',

View File

@ -139,20 +139,14 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args):
ticker_interval = strategy.ticker_interval ticker_interval = strategy.ticker_interval
timerange = Arguments.parse_timerange(args.timerange) timerange = Arguments.parse_timerange(args.timerange)
tickers = {}
if args.live:
logger.info('Downloading pairs.')
exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs])
for pair in pairs:
tickers[pair] = exchange.klines((pair, ticker_interval))
else:
tickers = history.load_data( tickers = history.load_data(
datadir=Path(str(_CONF.get("datadir"))), datadir=Path(str(_CONF.get("datadir"))),
pairs=pairs, pairs=pairs,
ticker_interval=ticker_interval, ticker_interval=ticker_interval,
refresh_pairs=_CONF.get('refresh_pairs', False), refresh_pairs=_CONF.get('refresh_pairs', False),
timerange=timerange, timerange=timerange,
exchange=Exchange(_CONF) exchange=Exchange(_CONF),
live=args.live,
) )
# No ticker found, impossible to download, len mismatch # No ticker found, impossible to download, len mismatch