adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2678 from hroff-1902/dataprovider-history-split-refresh

Browse Source 
Dataprovider history: split refresh part
This commit is contained in:
hroff-1902 2019-12-18 00:30:47 +03:00 committed by GitHub
commit 021fa1ca1a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 62 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
freqtrade/data/history.py
View File

@ -128,39 +128,26 @@ def load_pair_history(pair: str,
timeframe: str,
datadir: Path,
timerange: Optional[TimeRange] = None,
refresh_pairs: bool = False,
exchange: Optional[Exchange] = None,
fill_up_missing: bool = True,
drop_incomplete: bool = True,
startup_candles: int = 0,
) -> DataFrame:
"""
Loads cached ticker history for the given pair.
Load cached ticker history for the given pair.
:param pair: Pair to load data for
:param timeframe: Ticker timeframe (e.g. "5m")
:param datadir: Path to the data storage location.
:param timerange: Limit data to be loaded to this timerange
:param refresh_pairs: Refresh pairs from exchange.
(Note: Requires exchange to be passed as well.)
:param exchange: Exchange object (needed when using "refresh_pairs")
:param fill_up_missing: Fill missing values with "No action"-candles
:param drop_incomplete: Drop last candle assuming it may be incomplete.
:param startup_candles: Additional candles to load at the start of the period
:return: DataFrame with ohlcv data, or empty DataFrame
"""
timerange_startup = deepcopy(timerange)
if startup_candles > 0 and timerange_startup:
timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles)
# The user forced the refresh of pairs
if refresh_pairs:
_download_pair_history(datadir=datadir,
exchange=exchange,
pair=pair,
timeframe=timeframe,
timerange=timerange)
pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup)
if pairdata:
@ -180,30 +167,22 @@ def load_pair_history(pair: str,
def load_data(datadir: Path,
timeframe: str,
pairs: List[str],
refresh_pairs: bool = False,
exchange: Optional[Exchange] = None,
timerange: Optional[TimeRange] = None,
fill_up_missing: bool = True,
startup_candles: int = 0,
fail_without_data: bool = False
) -> Dict[str, DataFrame]:
"""
Loads ticker history data for a list of pairs
Load ticker history data for a list of pairs.
:param datadir: Path to the data storage location.
:param timeframe: Ticker Timeframe (e.g. "5m")
:param pairs: List of pairs to load
:param refresh_pairs: Refresh pairs from exchange.
(Note: Requires exchange to be passed as well.)
:param exchange: Exchange object (needed when using "refresh_pairs")
:param timerange: Limit data to be loaded to this timerange
:param fill_up_missing: Fill missing values with "No action"-candles
:param startup_candles: Additional candles to load at the start of the period
:param fail_without_data: Raise OperationalException if no data is found.
:return: dict(<pair>:<Dataframe>)
TODO: refresh_pairs is still used by edge to keep the data uptodate.
This should be replaced in the future. Instead, writing the current candles to disk
from dataprovider should be implemented, as this would avoid loading ohlcv data twice.
exchange and refresh_pairs are then not needed here nor in load_pair_history.
"""
result: Dict[str, DataFrame] = {}
if startup_candles > 0 and timerange:
@ -212,8 +191,6 @@ def load_data(datadir: Path,
for pair in pairs:
hist = load_pair_history(pair=pair, timeframe=timeframe,
datadir=datadir, timerange=timerange,
refresh_pairs=refresh_pairs,
exchange=exchange,
fill_up_missing=fill_up_missing,
startup_candles=startup_candles)
if not hist.empty:
@ -224,6 +201,27 @@ def load_data(datadir: Path,
return result
def refresh_data(datadir: Path,
timeframe: str,
pairs: List[str],
exchange: Exchange,
timerange: Optional[TimeRange] = None,
) -> None:
"""
Refresh ticker history data for a list of pairs.
:param datadir: Path to the data storage location.
:param timeframe: Ticker Timeframe (e.g. "5m")
:param pairs: List of pairs to load
:param exchange: Exchange object
:param timerange: Limit data to be loaded to this timerange
"""
for pair in pairs:
_download_pair_history(pair=pair, timeframe=timeframe,
datadir=datadir, timerange=timerange,
exchange=exchange)
def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path:
pair_s = pair.replace("/", "_")
filename = datadir.joinpath(f'{pair_s}-{timeframe}.json')

11
freqtrade/edge/__init__.py
View File

@ -94,12 +94,19 @@ class Edge:
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
logger.info('Using local backtesting data (using whitelist in given config) ...')
if self._refresh_pairs:
history.refresh_data(
datadir=Path(self.config['datadir']),
pairs=pairs,
exchange=self.exchange,
timeframe=self.strategy.ticker_interval,
timerange=self._timerange,
)
data = history.load_data(
datadir=Path(self.config['datadir']),
pairs=pairs,
timeframe=self.strategy.ticker_interval,
refresh_pairs=self._refresh_pairs,
exchange=self.exchange,
timerange=self._timerange,
startup_candles=self.strategy.startup_candle_count,
)

28
tests/data/test_history.py
View File

@ -21,6 +21,7 @@ from freqtrade.data.history import (_download_pair_history,
pair_trades_filename,
refresh_backtest_ohlcv_data,
refresh_backtest_trades_data,
refresh_data,
trim_dataframe, trim_tickerlist,
validate_backtest_data)
from freqtrade.exchange import timeframe_to_minutes
@ -129,16 +130,17 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog,
)
# download a new pair if refresh_pairs is set
load_pair_history(datadir=testdatadir, timeframe='1m',
refresh_pairs=True, exchange=exchange, pair='MEME/BTC')
refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'],
exchange=exchange)
load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC')
assert file.is_file()
assert log_has_re(
'Download history data for pair: "MEME/BTC", timeframe: 1m '
'and store in .*', caplog
)
with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'):
load_pair_history(datadir=testdatadir, timeframe='1m',
refresh_pairs=True, exchange=None, pair='MEME/BTC')
refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'],
exchange=None)
_clean_test_file(file)
@ -372,12 +374,24 @@ def test_load_partial_missing(testdatadir, caplog) -> None:
def test_init(default_conf, mocker) -> None:
exchange = get_patched_exchange(mocker, default_conf)
assert {} == load_data(
datadir='',
exchange=exchange,
pairs=[],
refresh_pairs=True,
timeframe=default_conf['ticker_interval']
)
def test_init_with_refresh(default_conf, mocker) -> None:
exchange = get_patched_exchange(mocker, default_conf)
refresh_data(
datadir='',
pairs=[],
timeframe=default_conf['ticker_interval'],
exchange=exchange
)
assert {} == load_data(
datadir='',
pairs=[],
timeframe=default_conf['ticker_interval']
)

7
tests/edge/test_edge.py
View File

@ -255,8 +255,8 @@ def test_edge_heartbeat_calculate(mocker, edge_conf):
assert edge.calculate() is False
def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False,
timerange=None, exchange=None, *args, **kwargs):
def mocked_load_data(datadir, pairs=[], timeframe='0m',
timerange=None, *args, **kwargs):
hz = 0.1
base = 0.001
@ -290,6 +290,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False,
def test_edge_process_downloaded_data(mocker, edge_conf):
freqtrade = get_patched_freqtradebot(mocker, edge_conf)
mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
mocker.patch('freqtrade.data.history.refresh_data', MagicMock())
mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
@ -301,6 +302,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf):
def test_edge_process_no_data(mocker, edge_conf, caplog):
freqtrade = get_patched_freqtradebot(mocker, edge_conf)
mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
mocker.patch('freqtrade.data.history.refresh_data', MagicMock())
mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={}))
edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
@ -313,6 +315,7 @@ def test_edge_process_no_data(mocker, edge_conf, caplog):
def test_edge_process_no_trades(mocker, edge_conf, caplog):
freqtrade = get_patched_freqtradebot(mocker, edge_conf)
mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
mocker.patch('freqtrade.data.history.refresh_data', MagicMock())
mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
# Return empty
mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', MagicMock(return_value=[]))

4
tests/optimize/test_backtesting.py
View File

@ -116,8 +116,8 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None:
assert len(results) == num_results
def mocked_load_data(datadir, pairs=[], timeframe='0m', refresh_pairs=False,
timerange=None, exchange=None, live=False, *args, **kwargs):
def mocked_load_data(datadir, pairs=[], timeframe='0m',
timerange=None, *args, **kwargs):
tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange)
pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC",
fill_missing=True)}