From 3232251fea8f05c309169c9449261dc720ea5366 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 15:01:27 +0200 Subject: [PATCH 1/2] Refactor downloading ohlcv from utils to history --- freqtrade/data/history.py | 29 +++++++++++++++++++++++++++++ freqtrade/utils.py | 32 ++++++++------------------------ 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 007357d9a..aff9f5c74 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -280,6 +280,35 @@ def download_pair_history(datadir: Optional[Path], return False +def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], + dl_path: Path, timerange: TimeRange, + erase=False) -> List[str]: + """ + Refresh stored ohlcv data for backtesting and hyperopt operations. + Used by freqtrade download-data + :return: Pairs not available + """ + pairs_not_available = [] + for pair in pairs: + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for ticker_interval in timeframes: + + dl_file = pair_data_filename(dl_path, pair, ticker_interval) + if erase and dl_file.exists(): + logger.info( + f'Deleting existing data for pair {pair}, interval {ticker_interval}.') + dl_file.unlink() + + logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=str(ticker_interval), + timerange=timerange) + return pairs_not_available + + def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ Get the maximum timeframe for the given backtest data diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 56e60ec82..162493a3f 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -2,13 +2,13 @@ import logging import sys from argparse import Namespace from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, List import arrow from freqtrade.configuration import Configuration, TimeRange from freqtrade.configuration.directory_operations import create_userdata_dir -from freqtrade.data.history import download_pair_history +from freqtrade.data.history import refresh_backtest_ohlcv_data from freqtrade.exchange import available_exchanges from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode @@ -75,39 +75,23 @@ def start_download_data(args: Namespace) -> None: logger.info(f'About to download pairs: {config["pairs"]}, ' f'intervals: {config["timeframes"]} to {dl_path}') - pairs_not_available = [] + pairs_not_available: List[str] = [] try: # Init exchange exchange = ExchangeResolver(config['exchange']['name'], config).exchange - for pair in config["pairs"]: - if pair not in exchange.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for ticker_interval in config["timeframes"]: - pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{ticker_interval}.json' - dl_file = dl_path.joinpath(filename) - if config.get("erase") and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {ticker_interval}.') - dl_file.unlink() - - logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, ticker_interval=str(ticker_interval), - timerange=timerange) + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config["pairs"], timeframes=config["timeframes"], + dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") finally: if pairs_not_available: - logger.info( - f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {config['exchange']['name']}.") + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {config['exchange']['name']}.") # configuration.resolve_pairs_list() print(config) From da7da2ce524e0bd41576251af7d1a48ee71184ee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Aug 2019 15:02:40 +0200 Subject: [PATCH 2/2] Change tests to split function --- freqtrade/tests/data/test_history.py | 43 +++++++++++++- freqtrade/tests/test_utils.py | 87 ++++++---------------------- 2 files changed, 61 insertions(+), 69 deletions(-) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 7360f3c1c..ec4a05e63 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -5,7 +5,7 @@ import os import uuid from pathlib import Path from shutil import copyfile -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import arrow import pytest @@ -17,6 +17,7 @@ from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, make_testdata_path, + refresh_backtest_ohlcv_data, trim_tickerlist) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json @@ -558,3 +559,43 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC', min_date, max_date, timeframe_to_minutes('5m')) assert len(caplog.record_tuples) == 0 + + +def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog): + dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + ex = get_patched_exchange(mocker, default_conf) + timerange = TimeRange.parse_timerange("20190101-20190102") + refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + timeframes=["1m", "5m"], dl_path=make_testdata_path(None), + timerange=timerange, erase=True + ) + + assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype == 'date' + + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + +def test_download_data_no_markets(mocker, default_conf, caplog): + dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + ex = get_patched_exchange(mocker, default_conf) + timerange = TimeRange.parse_timerange("20190101-20190102") + unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + timeframes=["1m", "5m"], + dl_path=make_testdata_path(None), + timerange=timerange, erase=False + ) + + assert dl_mock.call_count == 0 + assert "ETH/BTC" in unav_pairs + assert "XRP/BTC" in unav_pairs + assert log_has("Skipping pair ETH/BTC...", caplog) diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index d04e62b28..9e09fd298 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -1,5 +1,4 @@ import re -from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pytest @@ -70,74 +69,8 @@ def test_create_datadir(caplog, mocker): assert len(caplog.record_tuples) == 0 -def test_download_data(mocker, markets, caplog): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) - patch_exchange(mocker) - mocker.patch( - 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) - ) - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - mocker.patch.object(Path, "unlink", MagicMock()) - - args = [ - "download-data", - "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC", - "--erase", - ] - start_download_data(get_args(args)) - - assert dl_mock.call_count == 4 - assert dl_mock.call_args[1]['timerange'].starttype is None - assert dl_mock.call_args[1]['timerange'].stoptype is None - assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog) - assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) - - -def test_download_data_days(mocker, markets, caplog): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) - patch_exchange(mocker) - mocker.patch( - 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) - ) - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - mocker.patch.object(Path, "unlink", MagicMock()) - - args = [ - "download-data", - "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC", - "--days", "20", - ] - - start_download_data(get_args(args)) - - assert dl_mock.call_count == 4 - assert dl_mock.call_args[1]['timerange'].starttype == 'date' - - assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) - - -def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) - patch_exchange(mocker) - mocker.patch( - 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) - ) - args = [ - "download-data", - "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC", - ] - start_download_data(get_args(args)) - - assert dl_mock.call_count == 0 - assert log_has("Skipping pair ETH/BTC...", caplog) - assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) - - def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.utils.download_pair_history', + dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -152,3 +85,21 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): start_download_data(get_args(args)) assert dl_mock.call_count == 1 + + +def test_download_data_no_markets(mocker, caplog): + dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20" + ] + start_download_data(get_args(args)) + assert dl_mock.call_args[1]['timerange'].starttype == "date" + assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)