Move data-stuff to data-commands

freqtrade/commands/__init__.py

@@ -1,10 +1,10 @@
-from .hyperopt_commands import (start_hyperopt_list, start_hyperopt_show)  # noqa: 401
+from freqtrade.commands.data_commands import start_download_data  # noqa: F401
-
+from freqtrade.commands.deploy_commands import (start_create_userdir,  # noqa: F401
-from .list_commands import (start_list_exchanges,  # noqa: F401
+                              start_new_hyperopt, start_new_strategy)
+from freqtrade.commands.hyperopt_commands import (start_hyperopt_list,  # noqa: F401
+                                start_hyperopt_show)
+from freqtrade.commands.list_commands import (start_list_exchanges,  # noqa: F401
                             start_list_markets, start_list_strategies,
                             start_list_timeframes)
-from .utils import setup_utils_configuration  # noqa: F401
+from freqtrade.commands.trade_commands import start_trading  # noqa: F401
-from .utils import (start_download_data,  # noqa: F401
+from freqtrade.commands.utils import setup_utils_configuration, start_test_pairlist  # noqa: F401
-                    start_test_pairlist)
-from .deploy_commands import (start_new_hyperopt, start_new_strategy, start_create_userdir)  # noqa: F401
-from .trade_commands import start_trading  # noqa: F401

freqtrade/commands/data_commands.py

@@ -0,0 +1,64 @@
+import logging
+import sys
+from typing import Any, Dict, List
+
+import arrow
+
+from freqtrade.configuration import TimeRange
+from freqtrade.data.history import (convert_trades_to_ohlcv,
+                                    refresh_backtest_ohlcv_data,
+                                    refresh_backtest_trades_data)
+from freqtrade.exceptions import OperationalException
+from freqtrade.resolvers import ExchangeResolver
+from freqtrade.state import RunMode
+from .utils import setup_utils_configuration
+
+logger = logging.getLogger(__name__)
+
+
+def start_download_data(args: Dict[str, Any]) -> None:
+    """
+    Download data (former download_backtest_data.py script)
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+    timerange = TimeRange()
+    if 'days' in config:
+        time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d")
+        timerange = TimeRange.parse_timerange(f'{time_since}-')
+
+    if 'pairs' not in config:
+        raise OperationalException(
+            "Downloading data requires a list of pairs. "
+            "Please check the documentation on how to configure this.")
+
+    logger.info(f'About to download pairs: {config["pairs"]}, '
+                f'intervals: {config["timeframes"]} to {config["datadir"]}')
+
+    pairs_not_available: List[str] = []
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
+    try:
+
+        if config.get('download_trades'):
+            pairs_not_available = refresh_backtest_trades_data(
+                exchange, pairs=config["pairs"], datadir=config['datadir'],
+                timerange=timerange, erase=config.get("erase"))
+
+            # Convert downloaded trade data to different timeframes
+            convert_trades_to_ohlcv(
+                pairs=config["pairs"], timeframes=config["timeframes"],
+                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"))
+        else:
+            pairs_not_available = refresh_backtest_ohlcv_data(
+                exchange, pairs=config["pairs"], timeframes=config["timeframes"],
+                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"))
+
+    except KeyboardInterrupt:
+        sys.exit("SIGINT received, aborting ...")
+
+    finally:
+        if pairs_not_available:
+            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
+                        f"on exchange {exchange.name}.")

freqtrade/commands/utils.py

@@ -1,30 +1,11 @@
-import csv
 import logging
-import sys
+from typing import Any, Dict
-from collections import OrderedDict
-from operator import itemgetter
-from pathlib import Path
-from typing import Any, Dict, List
 
-import arrow
 import rapidjson
-from colorama import init as colorama_init
-from tabulate import tabulate
 
-from freqtrade.configuration import (Configuration, TimeRange,
+from freqtrade.configuration import (Configuration, remove_credentials,
-                                     remove_credentials,
                                      validate_config_consistency)
-from freqtrade.configuration.directory_operations import (copy_sample_files,
+from freqtrade.resolvers import ExchangeResolver
-                                                          create_userdata_dir)
-from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
-from freqtrade.data.history import (convert_trades_to_ohlcv,
-                                    refresh_backtest_ohlcv_data,
-                                    refresh_backtest_trades_data)
-from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
-                                market_is_active, symbol_is_pair)
-from freqtrade.misc import plural, render_template
-from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.state import RunMode
 
 logger = logging.getLogger(__name__)
@@ -46,54 +27,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
     return config
 
 
-def start_download_data(args: Dict[str, Any]) -> None:
-    """
-    Download data (former download_backtest_data.py script)
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
-
-    timerange = TimeRange()
-    if 'days' in config:
-        time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d")
-        timerange = TimeRange.parse_timerange(f'{time_since}-')
-
-    if 'pairs' not in config:
-        raise OperationalException(
-            "Downloading data requires a list of pairs. "
-            "Please check the documentation on how to configure this.")
-
-    logger.info(f'About to download pairs: {config["pairs"]}, '
-                f'intervals: {config["timeframes"]} to {config["datadir"]}')
-
-    pairs_not_available: List[str] = []
-
-    # Init exchange
-    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
-    try:
-
-        if config.get('download_trades'):
-            pairs_not_available = refresh_backtest_trades_data(
-                exchange, pairs=config["pairs"], datadir=config['datadir'],
-                timerange=timerange, erase=config.get("erase"))
-
-            # Convert downloaded trade data to different timeframes
-            convert_trades_to_ohlcv(
-                pairs=config["pairs"], timeframes=config["timeframes"],
-                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"))
-        else:
-            pairs_not_available = refresh_backtest_ohlcv_data(
-                exchange, pairs=config["pairs"], timeframes=config["timeframes"],
-                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"))
-
-    except KeyboardInterrupt:
-        sys.exit("SIGINT received, aborting ...")
-
-    finally:
-        if pairs_not_available:
-            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
-                        f"on exchange {exchange.name}.")
-
-
 def start_test_pairlist(args: Dict[str, Any]) -> None:
     """
     Test Pairlist configuration

tests/test_utils.py

@@ -538,7 +538,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog):
 
 
 def test_download_data_keyboardInterrupt(mocker, caplog, markets):
-    dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data',
+    dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                            MagicMock(side_effect=KeyboardInterrupt))
     patch_exchange(mocker)
     mocker.patch(
@@ -556,7 +556,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets):
 
 
 def test_download_data_no_markets(mocker, caplog):
-    dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data',
+    dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                            MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker, id='binance')
     mocker.patch(
@@ -574,7 +574,7 @@ def test_download_data_no_markets(mocker, caplog):
 
 
 def test_download_data_no_exchange(mocker, caplog):
-    mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data',
+    mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                  MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker)
     mocker.patch(
@@ -594,7 +594,7 @@ def test_download_data_no_pairs(mocker, caplog):
 
     mocker.patch.object(Path, "exists", MagicMock(return_value=False))
 
-    mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data',
+    mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                  MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker)
     mocker.patch(
@@ -613,9 +613,9 @@ def test_download_data_no_pairs(mocker, caplog):
 
 
 def test_download_data_trades(mocker, caplog):
-    dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_trades_data',
+    dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_trades_data',
                            MagicMock(return_value=[]))
-    convert_mock = mocker.patch('freqtrade.commands.utils.convert_trades_to_ohlcv',
+    convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
                                 MagicMock(return_value=[]))
     patch_exchange(mocker)
     mocker.patch(