Move convert functions to convert module

freqtrade/configuration/arguments.py

@@ -47,7 +47,7 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
 
 ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"]
 
-ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to"]
+ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
 ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",

freqtrade/data/converter.py

@@ -3,6 +3,7 @@ Functions to convert data from one format to another
 """
 import logging
 from datetime import datetime, timezone
+from typing import Any, Dict
 
 import pandas as pd
 from pandas import DataFrame, to_datetime
@@ -175,3 +176,64 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
     # Drop 0 volume rows
     df_new = df_new.dropna()
     return df_new[DEFAULT_DATAFRAME_COLUMNS]
+
+
+def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool):
+    """
+    Convert trades from one format to another format.
+    :param config: Config dictionary
+    :param convert_from: Source format
+    :param convert_to: Target format
+    :param erase: Erase souce data (does not apply if source and target format are identical)
+    """
+    from freqtrade.data.history.idatahandler import get_datahandler
+    src = get_datahandler(config['datadir'], convert_from)
+    trg = get_datahandler(config['datadir'], convert_to)
+
+    if 'pairs' not in config:
+        config['pairs'] = src.trades_get_pairs(config['datadir'])
+    logger.info(f"Converting trades for {config['pairs']}")
+
+    for pair in config['pairs']:
+        data = src.trades_load(pair=pair)
+        logger.info(f"Converting {len(data)} trades for {pair}")
+        trg.trades_store(pair, data)
+        if erase and convert_from != convert_to:
+            logger.info(f"Deleting source Trade data for {pair}.")
+            src.trades_purge(pair=pair)
+
+
+def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool):
+    """
+    Convert ohlcv from one format to another format.
+    :param config: Config dictionary
+    :param convert_from: Source format
+    :param convert_to: Target format
+    :param erase: Erase souce data (does not apply if source and target format are identical)
+    """
+    from freqtrade.data.history.idatahandler import get_datahandler
+    src = get_datahandler(config['datadir'], convert_from)
+    trg = get_datahandler(config['datadir'], convert_to)
+    timeframes = config.get('timeframes', [config.get('ticker_interval')])
+    logger.info(f"Converting OHLCV for timeframe {timeframes}")
+
+    if 'pairs' not in config:
+        config['pairs'] = []
+        # Check timeframes or fall back to ticker_interval.
+        for timeframe in timeframes:
+            config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'],
+                                                       timeframe))
+    logger.info(f"Converting OHLCV for {config['pairs']}")
+
+    for timeframe in timeframes:
+        for pair in config['pairs']:
+            data = src.ohlcv_load(pair=pair, timeframe=timeframe,
+                                  timerange=None,
+                                  fill_missing=False,
+                                  drop_incomplete=False,
+                                  startup_candles=0)
+            logger.info(f"Converting {len(data)} candles for {pair}")
+            trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data)
+            if erase and convert_from != convert_to:
+                logger.info(f"Deleting source data for {pair} / {timeframe}")
+                src.ohlcv_purge(pair=pair, timeframe=timeframe)

freqtrade/data/history/jsondatahandler.py

@@ -62,6 +62,8 @@ class JsonDataHandler(IDataHandler):
         :param pair: Pair to load data
         :param timeframe: Ticker timeframe (e.g. "5m")
         :param timerange: Limit data to be loaded to this timerange.
+                        Optionally implemented by subclasses to avoid loading
+                        all data where possible.
         :return: DataFrame with ohlcv data, or empty DataFrame
         """
         filename = self._pair_data_filename(self._datadir, pair, timeframe)

freqtrade/utils.py

@@ -17,8 +17,9 @@ from freqtrade.configuration import (Configuration, TimeRange,
 from freqtrade.configuration.directory_operations import (copy_sample_files,
                                                           create_userdata_dir)
 from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
+from freqtrade.data.converter import (convert_ohlcv_format,
+                                      convert_trades_format)
 from freqtrade.data.history import (convert_trades_to_ohlcv,
-                                    get_datahandlerclass,
                                     refresh_backtest_ohlcv_data,
                                     refresh_backtest_trades_data)
 from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
@@ -248,55 +249,6 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
         print(tabulate(strats_to_print, headers='keys', tablefmt='pipe'))
 
 
-def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str):
-    """
-    TODO: move this to converter.py (?)
-    """
-    SrcClass = get_datahandlerclass(convert_from)
-    TrgClass = get_datahandlerclass(convert_to)
-
-    if 'pairs' not in config:
-        config['pairs'] = SrcClass.trades_get_pairs(config['datadir'])
-    logger.info(f"Converting trades for {config['pairs']}")
-    src = SrcClass(config['datadir'])
-    trg = TrgClass(config['datadir'])
-    for pair in config['pairs']:
-        data = src.trades_load(pair=pair)
-        logger.info(f"Converting {len(data)} trades for {pair}")
-        trg.trades_store(pair, data)
-
-
-def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str):
-    """
-    TODO: move this to converter.py (?)
-    """
-    SrcClass = get_datahandlerclass(convert_from)
-    TrgClass = get_datahandlerclass(convert_to)
-    timeframes = config.get('timeframes', [config.get('ticker_interval')])
-    logger.info(f"Converting OHLCV for timeframe {timeframes}")
-
-    if 'pairs' not in config:
-        config['pairs'] = []
-        # Check timeframes or fall back to ticker_interval.
-        for timeframe in timeframes:
-            config['pairs'].extend(SrcClass.ohlcv_get_pairs(config['datadir'],
-                                                            timeframe))
-    logger.info(f"Converting OHLCV for {config['pairs']}")
-
-    src = SrcClass(config['datadir'])
-    trg = TrgClass(config['datadir'])
-
-    for timeframe in timeframes:
-        for pair in config['pairs']:
-            data = src.ohlcv_load(pair=pair, timeframe=timeframe,
-                                  timerange=None,
-                                  fill_missing=False,
-                                  drop_incomplete=False,
-                                  startup_candles=0)
-            logger.info(f"Converting {len(data)} candles for {pair}")
-            trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data)
-
-
 def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     """
     Convert data from one format to another
@@ -304,10 +256,12 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     if ohlcv:
         convert_ohlcv_format(config,
-                             convert_from=args['format_from'], convert_to=args['format_to'])
+                             convert_from=args['format_from'], convert_to=args['format_to'],
+                             erase=args['erase'])
     else:
         convert_trades_format(config,
-                              convert_from=args['format_from'], convert_to=args['format_to'])
+                              convert_from=args['format_from'], convert_to=args['format_to'],
+                              erase=args['erase'])
 
 
 def start_list_timeframes(args: Dict[str, Any]) -> None:

tests/data/test_history.py

@@ -463,7 +463,8 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
 
 
 def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
-    dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', MagicMock())
+    dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history',
+                           MagicMock())
 
     ex = get_patched_exchange(mocker, default_conf)
     mocker.patch(