From 7baf11a497476f2b4b56d72f4ca6ac3ff9f37a91 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 07:04:53 +0100 Subject: [PATCH] Futures candles should go into a subdirectory --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/data_commands.py | 3 ++- freqtrade/data/history/hdf5datahandler.py | 5 ++++- freqtrade/data/history/idatahandler.py | 13 ++++++++++++- freqtrade/data/history/jsondatahandler.py | 6 +++++- freqtrade/rpc/api_server/api_v1.py | 3 ++- 6 files changed, 26 insertions(+), 6 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 3ebad54dc..711a2c4a2 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -65,7 +65,7 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] -ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] +ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive", "timerange", "download_trades", "exchange", "timeframes", diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 2737e1013..a75b4bfd2 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -156,7 +156,8 @@ def start_list_data(args: Dict[str, Any]) -> None: from freqtrade.data.history.idatahandler import get_datahandler dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv']) - paircombs = dhc.ohlcv_get_available_data(config['datadir']) + # TODO-lev: trading-mode should be parsed at config level, and available as Enum in the config. + paircombs = dhc.ohlcv_get_available_data(config['datadir'], config.get('trading_mode', 'spot')) if args['pairs']: paircombs = [comb for comb in paircombs if comb[0] in args['pairs']] diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 221758020..a390e887a 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -21,12 +21,15 @@ class HDF5DataHandler(IDataHandler): _columns = DEFAULT_DATAFRAME_COLUMNS @classmethod - def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + def ohlcv_get_available_data(cls, datadir: Path, trading_mode: str) -> ListPairsWithTimeframes: """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files + :param trading_mode: trading-mode to be used :return: List of Tuples of (pair, timeframe) """ + if trading_mode != 'spot': + datadir = datadir.joinpath('futures') _tmp = [ re.search( cls._OHLCV_REGEX, p.name diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index f616d143d..2d7df6ca5 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -38,10 +38,11 @@ class IDataHandler(ABC): raise NotImplementedError() @abstractclassmethod - def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + def ohlcv_get_available_data(cls, datadir: Path, trading_mode: str) -> ListPairsWithTimeframes: """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files + :param trading_mode: trading-mode to be used :return: List of Tuples of (pair, timeframe) """ @@ -178,6 +179,15 @@ class IDataHandler(ABC): """ return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + @classmethod + def create_dir_if_needed(cls, datadir: Path): + """ + Creates datadir if necessary + should only create directories for "futures" mode at the moment. + """ + if not datadir.parent.is_dir(): + datadir.parent.mkdir() + @classmethod def _pair_data_filename( cls, @@ -188,6 +198,7 @@ class IDataHandler(ABC): ) -> Path: pair_s = misc.pair_to_filename(pair) if candle_type: + datadir = datadir.joinpath('futures') candle_type = f"-{candle_type}" filename = datadir.joinpath(f'{pair_s}-{timeframe}{candle_type}.{cls._get_file_extension()}') return filename diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 32f6a0b1d..deacda338 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -23,12 +23,15 @@ class JsonDataHandler(IDataHandler): _columns = DEFAULT_DATAFRAME_COLUMNS @classmethod - def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + def ohlcv_get_available_data(cls, datadir: Path, trading_mode: str) -> ListPairsWithTimeframes: """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files + :param trading_mode: trading-mode to be used :return: List of Tuples of (pair, timeframe) """ + if trading_mode != 'spot': + datadir = datadir.joinpath('futures') _tmp = [ re.search( cls._OHLCV_REGEX, p.name @@ -74,6 +77,7 @@ class JsonDataHandler(IDataHandler): :return: None """ filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int _data['date'] = _data['date'].view(np.int64) // 1000 // 1000 diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 235cf6de3..f97d49e96 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -254,7 +254,8 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None)) - pair_interval = dh.ohlcv_get_available_data(config['datadir']) + # TODO-lev: xmatt to decide: use candle-type or market mode for this endpoint?? + pair_interval = dh.ohlcv_get_available_data(config['datadir'], 'spot') if timeframe: pair_interval = [pair for pair in pair_interval if pair[1] == timeframe]