Merge pull request #2180 from freqtrade/refactor_download

[Refactor] Logic for download-data to history
This commit is contained in:
Matthias 2019-08-26 06:13:19 +02:00 committed by GitHub
commit 4fcfb1eaca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 98 additions and 93 deletions

View File

@ -280,6 +280,35 @@ def download_pair_history(datadir: Optional[Path],
return False return False
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
dl_path: Path, timerange: TimeRange,
erase=False) -> List[str]:
"""
Refresh stored ohlcv data for backtesting and hyperopt operations.
Used by freqtrade download-data
:return: Pairs not available
"""
pairs_not_available = []
for pair in pairs:
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
for ticker_interval in timeframes:
dl_file = pair_data_filename(dl_path, pair, ticker_interval)
if erase and dl_file.exists():
logger.info(
f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
dl_file.unlink()
logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair, ticker_interval=str(ticker_interval),
timerange=timerange)
return pairs_not_available
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
""" """
Get the maximum timeframe for the given backtest data Get the maximum timeframe for the given backtest data

View File

@ -5,7 +5,7 @@ import os
import uuid import uuid
from pathlib import Path from pathlib import Path
from shutil import copyfile from shutil import copyfile
from unittest.mock import MagicMock from unittest.mock import MagicMock, PropertyMock
import arrow import arrow
import pytest import pytest
@ -17,6 +17,7 @@ from freqtrade.data import history
from freqtrade.data.history import (download_pair_history, from freqtrade.data.history import (download_pair_history,
load_cached_data_for_updating, load_cached_data_for_updating,
load_tickerdata_file, make_testdata_path, load_tickerdata_file, make_testdata_path,
refresh_backtest_ohlcv_data,
trim_tickerlist) trim_tickerlist)
from freqtrade.exchange import timeframe_to_minutes from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import file_dump_json from freqtrade.misc import file_dump_json
@ -558,3 +559,43 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None:
assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
min_date, max_date, timeframe_to_minutes('5m')) min_date, max_date, timeframe_to_minutes('5m'))
assert len(caplog.record_tuples) == 0 assert len(caplog.record_tuples) == 0
def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
mocker.patch.object(Path, "unlink", MagicMock())
ex = get_patched_exchange(mocker, default_conf)
timerange = TimeRange.parse_timerange("20190101-20190102")
refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
timeframes=["1m", "5m"], dl_path=make_testdata_path(None),
timerange=timerange, erase=True
)
assert dl_mock.call_count == 4
assert dl_mock.call_args[1]['timerange'].starttype == 'date'
assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
def test_download_data_no_markets(mocker, default_conf, caplog):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
)
ex = get_patched_exchange(mocker, default_conf)
timerange = TimeRange.parse_timerange("20190101-20190102")
unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
timeframes=["1m", "5m"],
dl_path=make_testdata_path(None),
timerange=timerange, erase=False
)
assert dl_mock.call_count == 0
assert "ETH/BTC" in unav_pairs
assert "XRP/BTC" in unav_pairs
assert log_has("Skipping pair ETH/BTC...", caplog)

View File

@ -1,5 +1,4 @@
import re import re
from pathlib import Path
from unittest.mock import MagicMock, PropertyMock from unittest.mock import MagicMock, PropertyMock
import pytest import pytest
@ -70,74 +69,8 @@ def test_create_datadir(caplog, mocker):
assert len(caplog.record_tuples) == 0 assert len(caplog.record_tuples) == 0
def test_download_data(mocker, markets, caplog):
dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
mocker.patch.object(Path, "unlink", MagicMock())
args = [
"download-data",
"--exchange", "binance",
"--pairs", "ETH/BTC", "XRP/BTC",
"--erase",
]
start_download_data(get_args(args))
assert dl_mock.call_count == 4
assert dl_mock.call_args[1]['timerange'].starttype is None
assert dl_mock.call_args[1]['timerange'].stoptype is None
assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog)
assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
def test_download_data_days(mocker, markets, caplog):
dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
mocker.patch.object(Path, "unlink", MagicMock())
args = [
"download-data",
"--exchange", "binance",
"--pairs", "ETH/BTC", "XRP/BTC",
"--days", "20",
]
start_download_data(get_args(args))
assert dl_mock.call_count == 4
assert dl_mock.call_args[1]['timerange'].starttype == 'date'
assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
def test_download_data_no_markets(mocker, caplog):
dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
)
args = [
"download-data",
"--exchange", "binance",
"--pairs", "ETH/BTC", "XRP/BTC",
]
start_download_data(get_args(args))
assert dl_mock.call_count == 0
assert log_has("Skipping pair ETH/BTC...", caplog)
assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)
def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_keyboardInterrupt(mocker, caplog, markets):
dl_mock = mocker.patch('freqtrade.utils.download_pair_history', dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
MagicMock(side_effect=KeyboardInterrupt)) MagicMock(side_effect=KeyboardInterrupt))
patch_exchange(mocker) patch_exchange(mocker)
mocker.patch( mocker.patch(
@ -152,3 +85,21 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets):
start_download_data(get_args(args)) start_download_data(get_args(args))
assert dl_mock.call_count == 1 assert dl_mock.call_count == 1
def test_download_data_no_markets(mocker, caplog):
dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
)
args = [
"download-data",
"--exchange", "binance",
"--pairs", "ETH/BTC", "XRP/BTC",
"--days", "20"
]
start_download_data(get_args(args))
assert dl_mock.call_args[1]['timerange'].starttype == "date"
assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)

View File

@ -2,13 +2,13 @@ import logging
import sys import sys
from argparse import Namespace from argparse import Namespace
from pathlib import Path from pathlib import Path
from typing import Any, Dict from typing import Any, Dict, List
import arrow import arrow
from freqtrade.configuration import Configuration, TimeRange from freqtrade.configuration import Configuration, TimeRange
from freqtrade.configuration.directory_operations import create_userdata_dir from freqtrade.configuration.directory_operations import create_userdata_dir
from freqtrade.data.history import download_pair_history from freqtrade.data.history import refresh_backtest_ohlcv_data
from freqtrade.exchange import available_exchanges from freqtrade.exchange import available_exchanges
from freqtrade.resolvers import ExchangeResolver from freqtrade.resolvers import ExchangeResolver
from freqtrade.state import RunMode from freqtrade.state import RunMode
@ -75,38 +75,22 @@ def start_download_data(args: Namespace) -> None:
logger.info(f'About to download pairs: {config["pairs"]}, ' logger.info(f'About to download pairs: {config["pairs"]}, '
f'intervals: {config["timeframes"]} to {dl_path}') f'intervals: {config["timeframes"]} to {dl_path}')
pairs_not_available = [] pairs_not_available: List[str] = []
try: try:
# Init exchange # Init exchange
exchange = ExchangeResolver(config['exchange']['name'], config).exchange exchange = ExchangeResolver(config['exchange']['name'], config).exchange
for pair in config["pairs"]: pairs_not_available = refresh_backtest_ohlcv_data(
if pair not in exchange.markets: exchange, pairs=config["pairs"], timeframes=config["timeframes"],
pairs_not_available.append(pair) dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
logger.info(f"Skipping pair {pair}...")
continue
for ticker_interval in config["timeframes"]:
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{ticker_interval}.json'
dl_file = dl_path.joinpath(filename)
if config.get("erase") and dl_file.exists():
logger.info(
f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
dl_file.unlink()
logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair, ticker_interval=str(ticker_interval),
timerange=timerange)
except KeyboardInterrupt: except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...") sys.exit("SIGINT received, aborting ...")
finally: finally:
if pairs_not_available: if pairs_not_available:
logger.info( logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {config['exchange']['name']}.") f"on exchange {config['exchange']['name']}.")
# configuration.resolve_pairs_list() # configuration.resolve_pairs_list()