diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 121bd23b4..2b9b362bf 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -47,7 +47,7 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] -ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to"] +ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 41a843e36..c6ec4fa2b 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -3,6 +3,7 @@ Functions to convert data from one format to another """ import logging from datetime import datetime, timezone +from typing import Any, Dict import pandas as pd from pandas import DataFrame, to_datetime @@ -175,3 +176,64 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame: # Drop 0 volume rows df_new = df_new.dropna() return df_new[DEFAULT_DATAFRAME_COLUMNS] + + +def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): + """ + Convert trades from one format to another format. + :param config: Config dictionary + :param convert_from: Source format + :param convert_to: Target format + :param erase: Erase souce data (does not apply if source and target format are identical) + """ + from freqtrade.data.history.idatahandler import get_datahandler + src = get_datahandler(config['datadir'], convert_from) + trg = get_datahandler(config['datadir'], convert_to) + + if 'pairs' not in config: + config['pairs'] = src.trades_get_pairs(config['datadir']) + logger.info(f"Converting trades for {config['pairs']}") + + for pair in config['pairs']: + data = src.trades_load(pair=pair) + logger.info(f"Converting {len(data)} trades for {pair}") + trg.trades_store(pair, data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source Trade data for {pair}.") + src.trades_purge(pair=pair) + + +def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): + """ + Convert ohlcv from one format to another format. + :param config: Config dictionary + :param convert_from: Source format + :param convert_to: Target format + :param erase: Erase souce data (does not apply if source and target format are identical) + """ + from freqtrade.data.history.idatahandler import get_datahandler + src = get_datahandler(config['datadir'], convert_from) + trg = get_datahandler(config['datadir'], convert_to) + timeframes = config.get('timeframes', [config.get('ticker_interval')]) + logger.info(f"Converting OHLCV for timeframe {timeframes}") + + if 'pairs' not in config: + config['pairs'] = [] + # Check timeframes or fall back to ticker_interval. + for timeframe in timeframes: + config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'], + timeframe)) + logger.info(f"Converting OHLCV for {config['pairs']}") + + for timeframe in timeframes: + for pair in config['pairs']: + data = src.ohlcv_load(pair=pair, timeframe=timeframe, + timerange=None, + fill_missing=False, + drop_incomplete=False, + startup_candles=0) + logger.info(f"Converting {len(data)} candles for {pair}") + trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source data for {pair} / {timeframe}") + src.ohlcv_purge(pair=pair, timeframe=timeframe) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index f40cf969f..14f643705 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -62,6 +62,8 @@ class JsonDataHandler(IDataHandler): :param pair: Pair to load data :param timeframe: Ticker timeframe (e.g. "5m") :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename(self._datadir, pair, timeframe) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 1bb0f611a..b0ef7241b 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -17,8 +17,9 @@ from freqtrade.configuration import (Configuration, TimeRange, from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY +from freqtrade.data.converter import (convert_ohlcv_format, + convert_trades_format) from freqtrade.data.history import (convert_trades_to_ohlcv, - get_datahandlerclass, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) from freqtrade.exchange import (available_exchanges, ccxt_exchanges, @@ -248,55 +249,6 @@ def start_list_strategies(args: Dict[str, Any]) -> None: print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) -def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str): - """ - TODO: move this to converter.py (?) - """ - SrcClass = get_datahandlerclass(convert_from) - TrgClass = get_datahandlerclass(convert_to) - - if 'pairs' not in config: - config['pairs'] = SrcClass.trades_get_pairs(config['datadir']) - logger.info(f"Converting trades for {config['pairs']}") - src = SrcClass(config['datadir']) - trg = TrgClass(config['datadir']) - for pair in config['pairs']: - data = src.trades_load(pair=pair) - logger.info(f"Converting {len(data)} trades for {pair}") - trg.trades_store(pair, data) - - -def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str): - """ - TODO: move this to converter.py (?) - """ - SrcClass = get_datahandlerclass(convert_from) - TrgClass = get_datahandlerclass(convert_to) - timeframes = config.get('timeframes', [config.get('ticker_interval')]) - logger.info(f"Converting OHLCV for timeframe {timeframes}") - - if 'pairs' not in config: - config['pairs'] = [] - # Check timeframes or fall back to ticker_interval. - for timeframe in timeframes: - config['pairs'].extend(SrcClass.ohlcv_get_pairs(config['datadir'], - timeframe)) - logger.info(f"Converting OHLCV for {config['pairs']}") - - src = SrcClass(config['datadir']) - trg = TrgClass(config['datadir']) - - for timeframe in timeframes: - for pair in config['pairs']: - data = src.ohlcv_load(pair=pair, timeframe=timeframe, - timerange=None, - fill_missing=False, - drop_incomplete=False, - startup_candles=0) - logger.info(f"Converting {len(data)} candles for {pair}") - trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) - - def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ Convert data from one format to another @@ -304,10 +256,12 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if ohlcv: convert_ohlcv_format(config, - convert_from=args['format_from'], convert_to=args['format_to']) + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) else: convert_trades_format(config, - convert_from=args['format_from'], convert_to=args['format_to']) + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) def start_list_timeframes(args: Dict[str, Any]) -> None: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index f84e819ad..404f52e87 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -463,7 +463,8 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', + MagicMock()) ex = get_patched_exchange(mocker, default_conf) mocker.patch(