Merge pull request #2180 from freqtrade/refactor_download

[Refactor] Logic for download-data to history

freqtrade/data/history.py

@@ -280,6 +280,35 @@ def download_pair_history(datadir: Optional[Path],
         return False
 
 
+def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
+                                dl_path: Path, timerange: TimeRange,
+                                erase=False) -> List[str]:
+    """
+    Refresh stored ohlcv data for backtesting and hyperopt operations.
+    Used by freqtrade download-data
+    :return: Pairs not available
+    """
+    pairs_not_available = []
+    for pair in pairs:
+        if pair not in exchange.markets:
+            pairs_not_available.append(pair)
+            logger.info(f"Skipping pair {pair}...")
+            continue
+        for ticker_interval in timeframes:
+
+            dl_file = pair_data_filename(dl_path, pair, ticker_interval)
+            if erase and dl_file.exists():
+                logger.info(
+                    f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
+                dl_file.unlink()
+
+            logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
+            download_pair_history(datadir=dl_path, exchange=exchange,
+                                  pair=pair, ticker_interval=str(ticker_interval),
+                                  timerange=timerange)
+    return pairs_not_available
+
+
 def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
     """
     Get the maximum timeframe for the given backtest data

freqtrade/tests/data/test_history.py

@@ -5,7 +5,7 @@ import os
 import uuid
 from pathlib import Path
 from shutil import copyfile
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, PropertyMock
 
 import arrow
 import pytest
@@ -17,6 +17,7 @@ from freqtrade.data import history
 from freqtrade.data.history import (download_pair_history,
                                     load_cached_data_for_updating,
                                     load_tickerdata_file, make_testdata_path,
+                                    refresh_backtest_ohlcv_data,
                                     trim_tickerlist)
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import file_dump_json
@@ -558,3 +559,43 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None:
     assert not history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
                                               min_date, max_date, timeframe_to_minutes('5m'))
     assert len(caplog.record_tuples) == 0
+
+
+def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog):
+    dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
+    )
+    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
+    mocker.patch.object(Path, "unlink", MagicMock())
+
+    ex = get_patched_exchange(mocker, default_conf)
+    timerange = TimeRange.parse_timerange("20190101-20190102")
+    refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
+                                timeframes=["1m", "5m"], dl_path=make_testdata_path(None),
+                                timerange=timerange, erase=True
+                                )
+
+    assert dl_mock.call_count == 4
+    assert dl_mock.call_args[1]['timerange'].starttype == 'date'
+
+    assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
+
+
+def test_download_data_no_markets(mocker, default_conf, caplog):
+    dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    ex = get_patched_exchange(mocker, default_conf)
+    timerange = TimeRange.parse_timerange("20190101-20190102")
+    unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
+                                             timeframes=["1m", "5m"],
+                                             dl_path=make_testdata_path(None),
+                                             timerange=timerange, erase=False
+                                             )
+
+    assert dl_mock.call_count == 0
+    assert "ETH/BTC" in unav_pairs
+    assert "XRP/BTC" in unav_pairs
+    assert log_has("Skipping pair ETH/BTC...", caplog)

freqtrade/tests/test_utils.py

@@ -1,5 +1,4 @@
 import re
-from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
 
 import pytest
@@ -70,74 +69,8 @@ def test_create_datadir(caplog, mocker):
     assert len(caplog.record_tuples) == 0
 
 
-def test_download_data(mocker, markets, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
-    patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    mocker.patch.object(Path, "unlink", MagicMock())
-
-    args = [
-        "download-data",
-        "--exchange", "binance",
-        "--pairs", "ETH/BTC", "XRP/BTC",
-        "--erase",
-    ]
-    start_download_data(get_args(args))
-
-    assert dl_mock.call_count == 4
-    assert dl_mock.call_args[1]['timerange'].starttype is None
-    assert dl_mock.call_args[1]['timerange'].stoptype is None
-    assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog)
-    assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
-
-
-def test_download_data_days(mocker, markets, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
-    patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
-    )
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    mocker.patch.object(Path, "unlink", MagicMock())
-
-    args = [
-        "download-data",
-        "--exchange", "binance",
-        "--pairs", "ETH/BTC", "XRP/BTC",
-        "--days", "20",
-    ]
-
-    start_download_data(get_args(args))
-
-    assert dl_mock.call_count == 4
-    assert dl_mock.call_args[1]['timerange'].starttype == 'date'
-
-    assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog)
-
-
-def test_download_data_no_markets(mocker, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock())
-    patch_exchange(mocker)
-    mocker.patch(
-        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
-    )
-    args = [
-        "download-data",
-        "--exchange", "binance",
-        "--pairs", "ETH/BTC", "XRP/BTC",
-    ]
-    start_download_data(get_args(args))
-
-    assert dl_mock.call_count == 0
-    assert log_has("Skipping pair ETH/BTC...", caplog)
-    assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)
-
-
 def test_download_data_keyboardInterrupt(mocker, caplog, markets):
-    dl_mock = mocker.patch('freqtrade.utils.download_pair_history',
+    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
                            MagicMock(side_effect=KeyboardInterrupt))
     patch_exchange(mocker)
     mocker.patch(
@@ -152,3 +85,21 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets):
         start_download_data(get_args(args))
 
     assert dl_mock.call_count == 1
+
+
+def test_download_data_no_markets(mocker, caplog):
+    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
+                           MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
+    patch_exchange(mocker)
+    mocker.patch(
+        'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})
+    )
+    args = [
+        "download-data",
+        "--exchange", "binance",
+        "--pairs", "ETH/BTC", "XRP/BTC",
+        "--days", "20"
+    ]
+    start_download_data(get_args(args))
+    assert dl_mock.call_args[1]['timerange'].starttype == "date"
+    assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog)

freqtrade/utils.py

@@ -2,13 +2,13 @@ import logging
 import sys
 from argparse import Namespace
 from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import arrow
 
 from freqtrade.configuration import Configuration, TimeRange
 from freqtrade.configuration.directory_operations import create_userdata_dir
-from freqtrade.data.history import download_pair_history
+from freqtrade.data.history import refresh_backtest_ohlcv_data
 from freqtrade.exchange import available_exchanges
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.state import RunMode
@@ -75,39 +75,23 @@ def start_download_data(args: Namespace) -> None:
     logger.info(f'About to download pairs: {config["pairs"]}, '
                 f'intervals: {config["timeframes"]} to {dl_path}')
 
-    pairs_not_available = []
+    pairs_not_available: List[str] = []
 
     try:
         # Init exchange
         exchange = ExchangeResolver(config['exchange']['name'], config).exchange
 
-        for pair in config["pairs"]:
+        pairs_not_available = refresh_backtest_ohlcv_data(
-            if pair not in exchange.markets:
+            exchange, pairs=config["pairs"], timeframes=config["timeframes"],
-                pairs_not_available.append(pair)
+            dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
-                logger.info(f"Skipping pair {pair}...")
-                continue
-            for ticker_interval in config["timeframes"]:
-                pair_print = pair.replace('/', '_')
-                filename = f'{pair_print}-{ticker_interval}.json'
-                dl_file = dl_path.joinpath(filename)
-                if config.get("erase") and dl_file.exists():
-                    logger.info(
-                        f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
-                    dl_file.unlink()
-
-                logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
-                download_pair_history(datadir=dl_path, exchange=exchange,
-                                      pair=pair, ticker_interval=str(ticker_interval),
-                                      timerange=timerange)
 
     except KeyboardInterrupt:
         sys.exit("SIGINT received, aborting ...")
 
     finally:
         if pairs_not_available:
-            logger.info(
+            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
-                f"Pairs [{','.join(pairs_not_available)}] not available "
+                        f"on exchange {config['exchange']['name']}.")
-                f"on exchange {config['exchange']['name']}.")
 
     # configuration.resolve_pairs_list()
     print(config)