From 5b3f907b0cdb4c04d57810c811726872e066d27f Mon Sep 17 00:00:00 2001 From: Wade Dyck Date: Mon, 27 Dec 2021 11:16:38 -0700 Subject: [PATCH 1/7] Fixes a download_data bug when in futures mode. When specifying multiple pairs to download, the json filenames were inconsistent due to the reassignment of candle_type. Also adds the candle_type being downloaded to a log message. --- freqtrade/data/history/history_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 57fdc4a14..c391dcb5c 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -198,8 +198,8 @@ def _download_pair_history(pair: str, *, try: logger.info( - f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} ' - f'and store in {datadir}.' + f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, ' + f'candle type: {candle_type} and store in {datadir}.' ) # data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange) @@ -286,17 +286,17 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price']) # All exchanges need FundingRate for futures trading. # The timeframe is aligned to the mark-price timeframe. - for candle_type in (CandleType.FUNDING_RATE, fr_candle_type): + for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type): # TODO: this could be in most parts to the above. if erase: - if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): + if data_handler.ohlcv_purge(pair, timeframe, candle_type=funding_candle_type): logger.info( f'Deleting existing data for pair {pair}, interval {timeframe}.') _download_pair_history(pair=pair, process=process, datadir=datadir, exchange=exchange, timerange=timerange, data_handler=data_handler, timeframe=str(timeframe), new_pairs_days=new_pairs_days, - candle_type=candle_type) + candle_type=funding_candle_type) return pairs_not_available From a5742b3bbce0fb1642339a72d805e029c3d62413 Mon Sep 17 00:00:00 2001 From: Wade Dyck Date: Mon, 27 Dec 2021 11:26:49 -0700 Subject: [PATCH 2/7] Fixes a failing test_history due to changed log message. --- tests/data/test_history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index d70d69080..351ae9919 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -148,8 +148,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC', candle_type=candle_type) assert file.is_file() assert log_has_re( - r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m ' - r'and store in .*', caplog + r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m, ' + r'candle type: spot and store in .*', caplog ) From 3d9360bb8cc51d36cfc7eca6e092f7ebcd80868d Mon Sep 17 00:00:00 2001 From: Wade Dyck Date: Mon, 27 Dec 2021 11:46:05 -0700 Subject: [PATCH 3/7] When backtesting, pass the candle_type to load_data. --- freqtrade/optimize/backtesting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3fe9a55f8..caf568faf 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -196,6 +196,7 @@ class Backtesting: startup_candles=self.required_startup, fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), + candle_type=self.config.get('candle_type_def', CandleType.SPOT) ) min_date, max_date = history.get_timerange(data) @@ -224,6 +225,7 @@ class Backtesting: startup_candles=0, fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), + candle_type=self.config.get('candle_type_def', CandleType.SPOT) ) else: self.detail_data = {} From 82cdfba49441e657cacb7d9b58adc5ae34a571ae Mon Sep 17 00:00:00 2001 From: Wade Dyck Date: Mon, 27 Dec 2021 12:48:42 -0700 Subject: [PATCH 4/7] Remove the guards against downloading data in futures mode. --- freqtrade/commands/data_commands.py | 2 -- tests/commands/test_commands.py | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 0c6f48088..2c435a6e0 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -64,8 +64,6 @@ def start_download_data(args: Dict[str, Any]) -> None: try: if config.get('download_trades'): - if config.get('trading_mode') == 'futures': - raise OperationalException("Trade download not supported for futures.") pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 2b5504324..4c5a83c9c 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -822,10 +822,8 @@ def test_download_data_trades(mocker, caplog): "--trading-mode", "futures", "--dl-trades" ] - with pytest.raises(OperationalException, - match="Trade download not supported for futures."): - start_download_data(get_args(args)) + start_download_data(get_args(args)) def test_start_convert_trades(mocker, caplog): From 5bb2d3baea7a553380754c4da6d04a11d669dddd Mon Sep 17 00:00:00 2001 From: Wade Dyck Date: Tue, 28 Dec 2021 11:35:17 -0700 Subject: [PATCH 5/7] Revert "Remove the guards against downloading data in futures mode." This reverts commit 82cdfba49441e657cacb7d9b58adc5ae34a571ae. --- freqtrade/commands/data_commands.py | 2 ++ tests/commands/test_commands.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 2c435a6e0..0c6f48088 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -64,6 +64,8 @@ def start_download_data(args: Dict[str, Any]) -> None: try: if config.get('download_trades'): + if config.get('trading_mode') == 'futures': + raise OperationalException("Trade download not supported for futures.") pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 4c5a83c9c..2b5504324 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -822,8 +822,10 @@ def test_download_data_trades(mocker, caplog): "--trading-mode", "futures", "--dl-trades" ] + with pytest.raises(OperationalException, + match="Trade download not supported for futures."): - start_download_data(get_args(args)) + start_download_data(get_args(args)) def test_start_convert_trades(mocker, caplog): From ac06da40e43f8289fff3a92ad94da82fed7a176c Mon Sep 17 00:00:00 2001 From: Wade Dyck Date: Tue, 28 Dec 2021 11:43:42 -0700 Subject: [PATCH 6/7] Explicitly set the trading-mode to spot for the --dl-trades download test. --- tests/commands/test_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 2b5504324..583d59894 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -808,6 +808,7 @@ def test_download_data_trades(mocker, caplog): "--exchange", "kraken", "--pairs", "ETH/BTC", "XRP/BTC", "--days", "20", + "--trading-mode", "spot", "--dl-trades" ] start_download_data(get_args(args)) From 73276f1351676256538bfcb74e2c3ac59887c6a5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Dec 2021 17:36:47 +0100 Subject: [PATCH 7/7] Remove default argument from "download trades" test --- tests/commands/test_commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 583d59894..2b5504324 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -808,7 +808,6 @@ def test_download_data_trades(mocker, caplog): "--exchange", "kraken", "--pairs", "ETH/BTC", "XRP/BTC", "--days", "20", - "--trading-mode", "spot", "--dl-trades" ] start_download_data(get_args(args))