adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1892 from freqtrade/ref/live_data

Browse Source 
refactor `--live` handling
This commit is contained in:
Matthias 2019-06-01 06:20:11 +02:00 committed by GitHub
commit f04089ef1e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 74 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
freqtrade/data/history.py
View File

@ -122,21 +122,34 @@ def load_data(datadir: Optional[Path],
refresh_pairs: bool = False,
exchange: Optional[Exchange] = None,
timerange: TimeRange = TimeRange(None, None, 0, 0),
fill_up_missing: bool = True) -> Dict[str, DataFrame]:
fill_up_missing: bool = True,
live: bool = False
) -> Dict[str, DataFrame]:
"""
Loads ticker history data for a list of pairs the given parameters
:return: dict(<pair>:<tickerlist>)
"""
result = {}
result: Dict[str, DataFrame] = {}
if live:
if exchange:
logger.info('Live: Downloading data for all defined pairs ...')
exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs])
result = {key[0]: value for key, value in exchange._klines.items() if value is not None}
else:
raise OperationalException(
"Exchange needs to be initialized when using live data."
)
else:
logger.info('Using local backtesting data ...')
for pair in pairs:
hist = load_pair_history(pair=pair, ticker_interval=ticker_interval,
datadir=datadir, timerange=timerange,
refresh_pairs=refresh_pairs,
exchange=exchange,
fill_up_missing=fill_up_missing)
if hist is not None:
result[pair] = hist
for pair in pairs:
hist = load_pair_history(pair=pair, ticker_interval=ticker_interval,
datadir=datadir, timerange=timerange,
refresh_pairs=refresh_pairs,
exchange=exchange,
fill_up_missing=fill_up_missing)
if hist is not None:
result[pair] = hist
return result

29
freqtrade/optimize/backtesting.py
View File

@ -401,24 +401,17 @@ class Backtesting(object):
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
if self.config.get('live'):
logger.info('Downloading data for all pairs in whitelist ...')
self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs])
data = {key[0]: value for key, value in self.exchange._klines.items()}
else:
logger.info('Using local backtesting data (using whitelist in given config) ...')
timerange = Arguments.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = history.load_data(
datadir=Path(self.config['datadir']) if self.config.get('datadir') else None,
pairs=pairs,
ticker_interval=self.ticker_interval,
refresh_pairs=self.config.get('refresh_pairs', False),
exchange=self.exchange,
timerange=timerange
)
timerange = Arguments.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = history.load_data(
datadir=Path(self.config['datadir']) if self.config.get('datadir') else None,
pairs=pairs,
ticker_interval=self.ticker_interval,
refresh_pairs=self.config.get('refresh_pairs', False),
exchange=self.exchange,
timerange=timerange,
live=self.config.get('live', False)
)
if not data:
logger.critical("No data found. Terminating.")

29
freqtrade/tests/data/test_history.py
View File

@ -5,6 +5,7 @@ import os
import uuid
from pathlib import Path
from shutil import copyfile
from unittest.mock import MagicMock
import arrow
import pytest
@ -20,7 +21,8 @@ from freqtrade.data.history import (download_pair_history,
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import file_dump_json
from freqtrade.strategy.default_strategy import DefaultStrategy
from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange
from freqtrade.tests.conftest import (get_patched_exchange, log_has,
patch_exchange)
# Change this if modifying UNITTEST/BTC testdatafile
_BTC_UNITTEST_LENGTH = 13681
@ -136,6 +138,31 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau
_clean_test_file(file)
def test_load_data_live(default_conf, mocker, caplog) -> None:
refresh_mock = MagicMock()
mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock)
exchange = get_patched_exchange(mocker, default_conf)
history.load_data(datadir=None, ticker_interval='5m',
pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
live=True,
exchange=exchange)
assert refresh_mock.call_count == 1
assert len(refresh_mock.call_args_list[0][0][0]) == 2
assert log_has('Live: Downloading data for all defined pairs ...', caplog.record_tuples)
def test_load_data_live_noexchange(default_conf, mocker, caplog) -> None:
with pytest.raises(OperationalException,
match=r'Exchange needs to be initialized when using live data.'):
history.load_data(datadir=None, ticker_interval='5m',
pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'],
exchange=None,
live=True,
)
def test_testdata_path() -> None:
assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None))

7
freqtrade/tests/optimize/test_backtesting.py
View File

@ -105,7 +105,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None:
def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False,
timerange=None, exchange=None):
timerange=None, exchange=None, live=False):
tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', fill_missing=True)}
return pairdata
@ -492,7 +492,6 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
backtesting.start()
# check the logs, that will contain the backtest result
exists = [
'Using local backtesting data (using whitelist in given config) ...',
'Using stake_currency: BTC ...',
'Using stake_amount: 0.001 ...',
'Backtesting with data from 2017-11-14T21:17:00+00:00 '
@ -857,7 +856,7 @@ def test_backtest_start_live(default_conf, mocker, caplog):
'Using data folder: freqtrade/tests/testdata ...',
'Using stake_currency: BTC ...',
'Using stake_amount: 0.001 ...',
'Downloading data for all pairs in whitelist ...',
'Live: Downloading data for all defined pairs ...',
'Backtesting with data from 2017-11-14T19:31:00+00:00 '
'up to 2017-11-14T22:58:00+00:00 (0 days)..',
'Parameter --enable-position-stacking detected ...'
@ -916,7 +915,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog):
'Using data folder: freqtrade/tests/testdata ...',
'Using stake_currency: BTC ...',
'Using stake_amount: 0.001 ...',
'Downloading data for all pairs in whitelist ...',
'Live: Downloading data for all defined pairs ...',
'Backtesting with data from 2017-11-14T19:31:00+00:00 '
'up to 2017-11-14T22:58:00+00:00 (0 days)..',
'Parameter --enable-position-stacking detected ...',

24
scripts/plot_dataframe.py
View File

@ -139,21 +139,15 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args):
ticker_interval = strategy.ticker_interval
timerange = Arguments.parse_timerange(args.timerange)
tickers = {}
if args.live:
logger.info('Downloading pairs.')
exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs])
for pair in pairs:
tickers[pair] = exchange.klines((pair, ticker_interval))
else:
tickers = history.load_data(
datadir=Path(str(_CONF.get("datadir"))),
pairs=pairs,
ticker_interval=ticker_interval,
refresh_pairs=_CONF.get('refresh_pairs', False),
timerange=timerange,
exchange=Exchange(_CONF)
)
tickers = history.load_data(
datadir=Path(str(_CONF.get("datadir"))),
pairs=pairs,
ticker_interval=ticker_interval,
refresh_pairs=_CONF.get('refresh_pairs', False),
timerange=timerange,
exchange=Exchange(_CONF),
live=args.live,
)
# No ticker found, impossible to download, len mismatch
for pair, data in tickers.copy().items():