adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2180 from freqtrade/refactor_download

Browse Source 
[Refactor] Logic for download-data to history
This commit is contained in:
Matthias 2019-08-26 06:13:19 +02:00 committed by GitHub
commit 4fcfb1eaca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 98 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
freqtrade/data/history.py
View File

@ -280,6 +280,35 @@ def download_pair_history(datadir: Optional[Path],
return False
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
dl_path: Path, timerange: TimeRange,
erase=False) -> List[str]:
"""
Refresh stored ohlcv data for backtesting and hyperopt operations.
Used by freqtrade download-data
:return: Pairs not available
"""
pairs_not_available = []
for pair in pairs:
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
for ticker_interval in timeframes:
dl_file = pair_data_filename(dl_path, pair, ticker_interval)
if erase and dl_file.exists():
logger.info(
f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
dl_file.unlink()
logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair, ticker_interval=str(ticker_interval),
timerange=timerange)
return pairs_not_available
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
"""
Get the maximum timeframe for the given backtest data

43
freqtrade/tests/data/test_history.py
View File

@ -5,7 +5,7 @@ import os
import uuid
from pathlib import Path
from shutil import copyfile
from unittest.mock import MagicMock
from unittest.mock import MagicMock, PropertyMock
import arrow
import pytest
@ -17,6 +17,7 @@ from freqtrade.data import history
from freqtrade.data.history import (download_pair_history,
load_cached_data_for_updating,
load_tickerdata_file, make_testdata_path,
refresh_backtest_ohlcv_data,
trim_tickerlist)
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import file_dump_json
@ -558,3 +559,43 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None:
assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
min_date, max_date, timeframe_to_minutes('5m'))
assert len(caplog.record_tuples) == 0
def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
mocker.patch.object(Path, "unlink", MagicMock())
ex = get_patched_exchange(mocker, default_conf)
timerange = TimeRange.parse_timerange("20190101-20190102")
refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
timeframes=["1m", "5m"], dl_path=make_testdata_path(None),
timerange=timerange, erase=True
)
assert dl_mock.call_count == 4
assert dl_mock.call_args[1]['timerange'].starttype == 'date'
assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
def test_download_data_no_markets(mocker, default_conf, caplog):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
)
ex = get_patched_exchange(mocker, default_conf)
timerange = TimeRange.parse_timerange("20190101-20190102")
unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
timeframes=["1m", "5m"],
dl_path=make_testdata_path(None),
timerange=timerange, erase=False
)
assert dl_mock.call_count == 0
assert "ETH/BTC" in unav_pairs
assert "XRP/BTC" in unav_pairs
assert log_has("Skipping pair ETH/BTC...", caplog)

87
freqtrade/tests/test_utils.py
View File

@ -1,5 +1,4 @@
import re
from pathlib import Path
from unittest.mock import MagicMock, PropertyMock
import pytest
@ -70,74 +69,8 @@ def test_create_datadir(caplog, mocker):
assert len(caplog.record_tuples) == 0
def test_download_data(mocker, markets, caplog):
dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
mocker.patch.object(Path, "unlink", MagicMock())
args = [
"download-data",
"--exchange", "binance",
"--pairs", "ETH/BTC", "XRP/BTC",
"--erase",
]
start_download_data(get_args(args))
assert dl_mock.call_count == 4
assert dl_mock.call_args[1]['timerange'].starttype is None
assert dl_mock.call_args[1]['timerange'].stoptype is None
assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog)
assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
def test_download_data_days(mocker, markets, caplog):
dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
mocker.patch.object(Path, "unlink", MagicMock())
args = [
"download-data",
"--exchange", "binance",
"--pairs", "ETH/BTC", "XRP/BTC",
"--days", "20",
]
start_download_data(get_args(args))
assert dl_mock.call_count == 4
assert dl_mock.call_args[1]['timerange'].starttype == 'date'
assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
def test_download_data_no_markets(mocker, caplog):
dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
)
args = [
"download-data",
"--exchange", "binance",
"--pairs", "ETH/BTC", "XRP/BTC",
]
start_download_data(get_args(args))
assert dl_mock.call_count == 0
assert log_has("Skipping pair ETH/BTC...", caplog)
assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)
def test_download_data_keyboardInterrupt(mocker, caplog, markets):
dl_mock = mocker.patch('freqtrade.utils.download_pair_history',
dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
MagicMock(side_effect=KeyboardInterrupt))
patch_exchange(mocker)
mocker.patch(
@ -152,3 +85,21 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets):
start_download_data(get_args(args))
assert dl_mock.call_count == 1
def test_download_data_no_markets(mocker, caplog):
dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
patch_exchange(mocker)
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
)
args = [
"download-data",
"--exchange", "binance",
"--pairs", "ETH/BTC", "XRP/BTC",
"--days", "20"
]
start_download_data(get_args(args))
assert dl_mock.call_args[1]['timerange'].starttype == "date"
assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)

30
freqtrade/utils.py
View File

@ -2,13 +2,13 @@ import logging
import sys
from argparse import Namespace
from pathlib import Path
from typing import Any, Dict
from typing import Any, Dict, List
import arrow
from freqtrade.configuration import Configuration, TimeRange
from freqtrade.configuration.directory_operations import create_userdata_dir
from freqtrade.data.history import download_pair_history
from freqtrade.data.history import refresh_backtest_ohlcv_data
from freqtrade.exchange import available_exchanges
from freqtrade.resolvers import ExchangeResolver
from freqtrade.state import RunMode
@ -75,38 +75,22 @@ def start_download_data(args: Namespace) -> None:
logger.info(f'About to download pairs: {config["pairs"]}, '
f'intervals: {config["timeframes"]} to {dl_path}')
pairs_not_available = []
pairs_not_available: List[str] = []
try:
# Init exchange
exchange = ExchangeResolver(config['exchange']['name'], config).exchange
for pair in config["pairs"]:
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
for ticker_interval in config["timeframes"]:
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{ticker_interval}.json'
dl_file = dl_path.joinpath(filename)
if config.get("erase") and dl_file.exists():
logger.info(
f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
dl_file.unlink()
logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair, ticker_interval=str(ticker_interval),
timerange=timerange)
pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=config["pairs"], timeframes=config["timeframes"],
dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
finally:
if pairs_not_available:
logger.info(
f"Pairs [{','.join(pairs_not_available)}] not available "
logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {config['exchange']['name']}.")
# configuration.resolve_pairs_list()