From 3c8387ab611df10c8a327b5196f4798541f67b10 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Fri, 1 Apr 2022 20:48:13 +0900 Subject: [PATCH 01/53] Add exchange id for binance Futures --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index b808096d2..420061050 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -87,7 +87,7 @@ When trading on Binance Futures market, orderbook must be used because there is Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. -* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`. +* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance` for spot market, and use `binanceusdm` or `binancecoinm` for Futures market. * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`. ## Kraken From da0688b6aa26f6eee36acef9ede4186b48d876c6 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 2 Apr 2022 03:20:21 +0900 Subject: [PATCH 02/53] Revert "Add exchange id for binance Futures" This reverts commit 3c8387ab611df10c8a327b5196f4798541f67b10. --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 420061050..b808096d2 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -87,7 +87,7 @@ When trading on Binance Futures market, orderbook must be used because there is Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. -* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance` for spot market, and use `binanceusdm` or `binancecoinm` for Futures market. +* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`. * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`. ## Kraken From 8a3c2c6cad8beef24900f1939e2fbdc7736c365d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 13 May 2022 19:32:52 +0530 Subject: [PATCH 03/53] Corrected docstring Discussed in Discord --- freqtrade/persistence/trade_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index bb8c03dd2..b1fff5cf3 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -656,7 +656,7 @@ class LocalTrade(): def recalc_open_trade_value(self) -> None: """ Recalculate open_trade_value. - Must be called whenever open_rate, fee_open or is_short is changed. + Must be called whenever open_rate, fee_open is changed. """ self.open_trade_value = self._calc_open_trade_value() From 71a80cab3a19771628e3c8b936f3479003894c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 13 May 2022 21:19:40 +0530 Subject: [PATCH 04/53] fixed variable naming style --- freqtrade/optimize/backtesting.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 45300b744..3f2c35cd5 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -538,33 +538,33 @@ class Backtesting: trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) try: - closerate = self._get_close_rate(row, trade, exit_, trade_dur) + close_rate = self._get_close_rate(row, trade, exit_, trade_dur) except ValueError: return None - # call the custom exit price,with default value as previous closerate - current_profit = trade.calc_profit_ratio(closerate) + # call the custom exit price,with default value as previous close_rate + current_profit = trade.calc_profit_ratio(close_rate) order_type = self.strategy.order_types['exit'] if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT): # Custom exit pricing only for exit-signals if order_type == 'limit': - closerate = strategy_safe_wrapper(self.strategy.custom_exit_price, - default_retval=closerate)( + close_rate = strategy_safe_wrapper(self.strategy.custom_exit_price, + default_retval=close_rate)( pair=trade.pair, trade=trade, current_time=exit_candle_time, - proposed_rate=closerate, current_profit=current_profit, + proposed_rate=close_rate, current_profit=current_profit, exit_tag=exit_.exit_reason) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately if trade.is_short: - closerate = min(closerate, row[HIGH_IDX]) + close_rate = min(close_rate, row[HIGH_IDX]) else: - closerate = max(closerate, row[LOW_IDX]) + close_rate = max(close_rate, row[LOW_IDX]) # Confirm trade exit: time_in_force = self.strategy.order_time_in_force['exit'] if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount, - rate=closerate, + rate=close_rate, time_in_force=time_in_force, sell_reason=exit_.exit_reason, # deprecated exit_reason=exit_.exit_reason, @@ -597,12 +597,12 @@ class Backtesting: side=trade.exit_side, order_type=order_type, status="open", - price=closerate, - average=closerate, + price=close_rate, + average=close_rate, amount=trade.amount, filled=0, remaining=trade.amount, - cost=trade.amount * closerate, + cost=trade.amount * close_rate, ) trade.orders.append(order) return trade From 9d13c8729257ddfb9e8a64daf6bd8b99a9010fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 13 May 2022 21:46:25 +0530 Subject: [PATCH 05/53] cleaned up backtesting Solves the [bug](https://github.com/freqtrade/freqtrade/runs/6425715015?check_suite_focus=true) --- tests/optimize/test_backtest_detail.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 18b4c3621..0441d4214 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -522,7 +522,7 @@ tc32 = BTContainer(data=[ trailing_stop_positive=0.03, trades=[ BTrade(exit_reason=ExitType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3, is_short=True) - ] +] ) # Test 33: trailing_stop should be triggered by low of next candle, without adjusting stoploss using @@ -662,7 +662,7 @@ tc41 = BTContainer(data=[ custom_entry_price=4000, trades=[ BTrade(exit_reason=ExitType.STOP_LOSS, open_tick=1, close_tick=1, is_short=True) - ] +] ) # Test 42: Custom-entry-price around candle low @@ -933,3 +933,5 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) assert res.open_date == _get_frame_time_from_offset(trade.open_tick) assert res.close_date == _get_frame_time_from_offset(trade.close_tick) assert res.is_short == trade.is_short + backtesting.cleanup() + del backtesting From 64670726a61e8f56593a734d86ba24c10dbbc25f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 13 May 2022 21:52:26 +0530 Subject: [PATCH 06/53] flake8 fix --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3f2c35cd5..20f34a0bb 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -548,7 +548,7 @@ class Backtesting: # Custom exit pricing only for exit-signals if order_type == 'limit': close_rate = strategy_safe_wrapper(self.strategy.custom_exit_price, - default_retval=close_rate)( + default_retval=close_rate)( pair=trade.pair, trade=trade, current_time=exit_candle_time, proposed_rate=close_rate, current_profit=current_profit, From 64668b11dab7530f7606e9a9deee458689faaefe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 09:10:38 +0200 Subject: [PATCH 07/53] add ohlcv_has_history - disabling kraken downloads --- freqtrade/commands/data_commands.py | 6 ++++++ freqtrade/exchange/exchange.py | 13 +++++++++---- freqtrade/exchange/kraken.py | 1 + tests/commands/test_commands.py | 17 +++++++++++++++++ tests/exchange/test_ccxt_compat.py | 2 +- tests/exchange/test_exchange.py | 7 +++++++ 6 files changed, 41 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index a2e2a100a..61a99782e 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -79,6 +79,12 @@ def start_download_data(args: Dict[str, Any]) -> None: data_format_trades=config['dataformat_trades'], ) else: + if not exchange._ft_has.get('ohlcv_has_history', True): + raise OperationalException( + f"Historic klines not available for {exchange.name}. " + "Please use `--dl-trades` instead for this exchange " + "(will unfortunately take a long time)." + ) pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 65b9fb628..8d74a8446 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -64,6 +64,7 @@ class Exchange: "time_in_force_parameter": "timeInForce", "ohlcv_params": {}, "ohlcv_candle_limit": 500, + "ohlcv_has_history": True, # Some exchanges (Kraken) don't provide history via ohlcv "ohlcv_partial_candle": True, "ohlcv_require_since": False, # Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency @@ -621,13 +622,17 @@ class Exchange: # Allow 5 calls to the exchange per pair required_candle_call_count = int( (candle_count / candle_limit) + (0 if candle_count % candle_limit == 0 else 1)) + if self._ft_has['ohlcv_has_history']: - if required_candle_call_count > 5: - # Only allow 5 calls per pair to somewhat limit the impact + if required_candle_call_count > 5: + # Only allow 5 calls per pair to somewhat limit the impact + raise OperationalException( + f"This strategy requires {startup_candles} candles to start, which is more than 5x " + f"the amount of candles {self.name} provides for {timeframe}.") + elif required_candle_call_count > 1: raise OperationalException( - f"This strategy requires {startup_candles} candles to start, which is more than 5x " + f"This strategy requires {startup_candles} candles to start, which is more than " f"the amount of candles {self.name} provides for {timeframe}.") - if required_candle_call_count > 1: logger.warning(f"Using {required_candle_call_count} calls to get OHLCV. " f"This can result in slower operations for the bot. Please check " diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 33a2c7f87..900f6c898 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -23,6 +23,7 @@ class Kraken(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, "ohlcv_candle_limit": 720, + "ohlcv_has_history": False, "trades_pagination": "id", "trades_pagination_arg": "since", "mark_ohlcv_timeframe": "4h", diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 0932f4362..b37edf9c7 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -835,6 +835,23 @@ def test_download_data_trades(mocker, caplog): start_download_data(pargs) +def test_download_data_data_invalid(mocker): + patch_exchange(mocker, id="kraken") + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + "--exchange", "kraken", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, match=r"Historic klines not available for .*"): + start_download_data(pargs) + + def test_start_convert_trades(mocker, caplog): convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index d8832bb71..ac7c8a528 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -219,7 +219,7 @@ class TestCCXTExchange(): assert len(l2['asks']) == next_limit assert len(l2['asks']) == next_limit - def test_fetch_ohlcv(self, exchange): + def test_ccxt_fetch_ohlcv(self, exchange): exchange, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] timeframe = EXCHANGES[exchangename]['timeframe'] diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 77a04ac6c..ed2a7b7ee 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1084,6 +1084,13 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog): with pytest.raises(OperationalException, match=r'This strategy requires 6000.*'): Exchange(default_conf) + # Emulate kraken mode + ex._ft_has['ohlcv_has_history'] = False + with pytest.raises(OperationalException, + match=r'This strategy requires 2500.*, ' + r'which is more than the amount.*'): + ex.validate_required_startup_candles(2500, '5m') + def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) From 111b04c9e65668067646265e614326f81aa1bf1c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 09:51:44 +0200 Subject: [PATCH 08/53] Okx - conditional candle-length --- freqtrade/exchange/exchange.py | 17 ++++++++++----- freqtrade/exchange/okx.py | 28 +++++++++++++++++++++++-- tests/exchange/test_ccxt_compat.py | 33 ++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 8d74a8446..864aa36e9 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -309,12 +309,15 @@ class Exchange: if self.log_responses: logger.info(f"API {endpoint}: {response}") - def ohlcv_candle_limit(self, timeframe: str) -> int: + def ohlcv_candle_limit( + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> int: """ Exchange ohlcv candle limit Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit :param timeframe: Timeframe to check + :param candle_type: Candle-type + :param since_ms: Candle-type :return: Candle limit as integer """ return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( @@ -616,7 +619,7 @@ class Exchange: Checks if required startup_candles is more than ohlcv_candle_limit(). Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. """ - candle_limit = self.ohlcv_candle_limit(timeframe) + candle_limit = self.ohlcv_candle_limit(timeframe, self._config['candle_type_def'], None) # Require one more candle - to account for the still open candle. candle_count = startup_candles + 1 # Allow 5 calls to the exchange per pair @@ -1708,7 +1711,8 @@ class Exchange: :param candle_type: Any of the enum CandleType (must match trading mode!) """ - one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) + one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit( + timeframe, candle_type, since_ms) logger.debug( "one_call: %s msecs (%s)", one_call, @@ -1744,7 +1748,8 @@ class Exchange: if (not since_ms and (self._ft_has["ohlcv_require_since"] or self.required_candle_call_count > 1)): # Multiple calls for one pair - to get more history - one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) + one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit( + timeframe, candle_type, since_ms) move_to = one_call * self.required_candle_call_count now = timeframe_to_next_date(timeframe) since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000) @@ -1862,7 +1867,9 @@ class Exchange: pair, timeframe, since_ms, s ) params = deepcopy(self._ft_has.get('ohlcv_params', {})) - candle_limit = self.ohlcv_candle_limit(timeframe) + candle_limit = self.ohlcv_candle_limit( + timeframe, candle_type=candle_type, since_ms=since_ms) + if candle_type != CandleType.SPOT: params.update({'price': candle_type}) if candle_type != CandleType.FUNDING_RATE: diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 654021182..5e24997d7 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -1,13 +1,16 @@ import logging -from typing import Dict, List, Tuple +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional, Tuple import ccxt from freqtrade.constants import BuySell from freqtrade.enums import MarginMode, TradingMode +from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier +from freqtrade.exchange.exchange import timeframe_to_minutes logger = logging.getLogger(__name__) @@ -20,7 +23,7 @@ class Okx(Exchange): """ _ft_has: Dict = { - "ohlcv_candle_limit": 100, + "ohlcv_candle_limit": 300, # Warning, special case with data prior to X months "mark_ohlcv_timeframe": "4h", "funding_fee_timeframe": "8h", } @@ -37,6 +40,27 @@ class Okx(Exchange): net_only = True + def ohlcv_candle_limit( + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> int: + """ + Exchange ohlcv candle limit + Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits + per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit + :param timeframe: Timeframe to check + :param candle_type: Candle-type + :param since_ms: Candle-type + :return: Candle limit as integer + """ + now = datetime.now(timezone.utc) + offset_mins = timeframe_to_minutes(timeframe) * self._ft_has['ohlcv_candle_limit'] + if since_ms and since_ms < ((now - timedelta(minutes=offset_mins)).timestamp() * 1000): + return 100 + if candle_type not in (CandleType.FUTURES, CandleType.SPOT): + return 100 + + return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( + timeframe, self._ft_has.get('ohlcv_candle_limit'))) + @retrier def additional_exchange_init(self) -> None: """ diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index ac7c8a528..ea9a166f6 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -13,6 +13,7 @@ import pytest from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date +from freqtrade.exchange.exchange import timeframe_to_msecs from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_default_conf_usdt @@ -236,6 +237,38 @@ class TestCCXTExchange(): now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2)) assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) + def test_ccxt__async_get_candle_history(self, exchange): + exchange, exchangename = exchange + # For some weired reason, this test returns random lengths for bittrex. + if not exchange._ft_has['ohlcv_has_history'] or exchangename == 'bittrex': + return + pair = EXCHANGES[exchangename]['pair'] + timeframe = EXCHANGES[exchangename]['timeframe'] + candle_type = CandleType.SPOT + timeframe_ms = timeframe_to_msecs(timeframe) + now = timeframe_to_prev_date( + timeframe, datetime.now(timezone.utc)) + for offset in (360, 120, 30, 10, 5, 2): + since = now - timedelta(days=offset) + since_ms = int(since.timestamp() * 1000) + + res = exchange.loop.run_until_complete(exchange._async_get_candle_history( + pair=pair, + timeframe=timeframe, + since_ms=since_ms, + candle_type=candle_type + ) + ) + assert res + assert res[0] == pair + assert res[1] == timeframe + assert res[2] == candle_type + candles = res[3] + candle_count = exchange.ohlcv_candle_limit(timeframe, candle_type, since_ms) * 0.9 + candle_count1 = (now.timestamp() * 1000 - since_ms) // timeframe_ms + assert len(candles) >= min(candle_count, candle_count1) + assert candles[0][0] == since_ms or (since_ms + timeframe_ms) + def test_ccxt_fetch_funding_rate_history(self, exchange_futures): exchange, exchangename = exchange_futures if not exchange: From bb1b283d9548ea58cee0d588d48e631a015f8297 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 13:27:36 +0200 Subject: [PATCH 09/53] Update some ohlcv_candle_limit calls --- freqtrade/exchange/exchange.py | 2 +- freqtrade/exchange/okx.py | 2 +- freqtrade/plugins/pairlist/AgeFilter.py | 9 +++++---- freqtrade/plugins/pairlist/VolatilityFilter.py | 6 +++--- freqtrade/plugins/pairlist/VolumePairList.py | 7 ++++--- freqtrade/plugins/pairlist/rangestabilityfilter.py | 6 +++--- tests/exchange/test_ccxt_compat.py | 3 ++- tests/exchange/test_exchange.py | 8 ++++---- 8 files changed, 23 insertions(+), 20 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 864aa36e9..d2a01f394 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -310,7 +310,7 @@ class Exchange: logger.info(f"API {endpoint}: {response}") def ohlcv_candle_limit( - self, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> int: + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: """ Exchange ohlcv candle limit Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 5e24997d7..6d25bb12b 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -41,7 +41,7 @@ class Okx(Exchange): net_only = True def ohlcv_candle_limit( - self, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> int: + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: """ Exchange ohlcv candle limit Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index bb6f75012..418c0f14e 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -32,18 +32,19 @@ class AgeFilter(IPairList): self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) + candle_limit = exchange.ohlcv_candle_limit('1d', self._config['candle_type_def']) if self._min_days_listed < 1: raise OperationalException("AgeFilter requires min_days_listed to be >= 1") - if self._min_days_listed > exchange.ohlcv_candle_limit('1d'): + if self._min_days_listed > candle_limit: raise OperationalException("AgeFilter requires min_days_listed to not exceed " "exchange max request size " - f"({exchange.ohlcv_candle_limit('1d')})") + f"({candle_limit})") if self._max_days_listed and self._max_days_listed <= self._min_days_listed: raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted") - if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'): + if self._max_days_listed and self._max_days_listed > candle_limit: raise OperationalException("AgeFilter requires max_days_listed to not exceed " "exchange max request size " - f"({exchange.ohlcv_candle_limit('1d')})") + f"({candle_limit})") @property def needstickers(self) -> bool: diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 6aa857c2c..bab44bdd1 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -38,12 +38,12 @@ class VolatilityFilter(IPairList): self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) + candle_limit = exchange.ohlcv_candle_limit('1d', self._config['candle_type_def']) if self._days < 1: raise OperationalException("VolatilityFilter requires lookback_days to be >= 1") - if self._days > exchange.ohlcv_candle_limit('1d'): + if self._days > candle_limit: raise OperationalException("VolatilityFilter requires lookback_days to not " - "exceed exchange max request size " - f"({exchange.ohlcv_candle_limit('1d')})") + f"exceed exchange max request size ({candle_limit})") @property def needstickers(self) -> bool: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 26e7d45be..cd16a46a3 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -84,12 +84,13 @@ class VolumePairList(IPairList): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') + candle_limit = exchange.ohlcv_candle_limit( + self._lookback_timeframe, self._config['candle_type_def']) if self._lookback_period < 0: raise OperationalException("VolumeFilter requires lookback_period to be >= 0") - if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe): + if self._lookback_period > candle_limit: raise OperationalException("VolumeFilter requires lookback_period to not " - "exceed exchange max request size " - f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})") + f"exceed exchange max request size ({candle_limit})") @property def needstickers(self) -> bool: diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index c9edfd13d..de016c3a6 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -33,12 +33,12 @@ class RangeStabilityFilter(IPairList): self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) + candle_limit = exchange.ohlcv_candle_limit('1d', self._config['candle_type_def']) if self._days < 1: raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1") - if self._days > exchange.ohlcv_candle_limit('1d'): + if self._days > candle_limit: raise OperationalException("RangeStabilityFilter requires lookback_days to not " - "exceed exchange max request size " - f"({exchange.ohlcv_candle_limit('1d')})") + f"exceed exchange max request size ({candle_limit})") @property def needstickers(self) -> bool: diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index ea9a166f6..e016873cb 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -232,7 +232,8 @@ class TestCCXTExchange(): assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf)) # assert len(exchange.klines(pair_tf)) > 200 # Assume 90% uptime ... - assert len(exchange.klines(pair_tf)) > exchange.ohlcv_candle_limit(timeframe) * 0.90 + assert len(exchange.klines(pair_tf)) > exchange.ohlcv_candle_limit( + timeframe, CandleType.SPOT) * 0.90 # Check if last-timeframe is within the last 2 intervals now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2)) assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ed2a7b7ee..9d7b77a8e 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1882,7 +1882,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls - since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 + since = 5 * 60 * exchange.ohlcv_candle_limit('5m', CandleType.SPOT) * 1.8 ret = exchange.get_historic_ohlcv( pair, "5m", @@ -1948,7 +1948,7 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name, candle_ty exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls - since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 + since = 5 * 60 * exchange.ohlcv_candle_limit('5m', CandleType.SPOT) * 1.8 ret = exchange.get_historic_ohlcv_as_df( pair, "5m", @@ -2002,7 +2002,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ ) # Required candles candles = (end_ts - start_ts) / 300_000 - exp = candles // exchange.ohlcv_candle_limit('5m') + 1 + exp = candles // exchange.ohlcv_candle_limit('5m', CandleType.SPOT) + 1 # Depending on the exchange, this should be called between 1 and 6 times. assert exchange._api_async.fetch_ohlcv.call_count == exp @@ -3349,7 +3349,7 @@ def test_ohlcv_candle_limit(default_conf, mocker, exchange_name): expected = exchange._ft_has['ohlcv_candle_limit_per_timeframe'][timeframe] # This should only run for bittrex assert exchange_name == 'bittrex' - assert exchange.ohlcv_candle_limit(timeframe) == expected + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT) == expected def test_timeframe_to_minutes(): From 2a1368d508a621a35579bed55db1d6eb841933ee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 14:14:53 +0200 Subject: [PATCH 10/53] Offsetfilter: add number_assets parameter closes #6824 --- docs/includes/pairlists.md | 8 ++++---- freqtrade/plugins/pairlist/OffsetFilter.py | 9 ++++++++- tests/plugins/test_pairlist.py | 10 +++++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 6acd361fe..0f55c1b79 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -160,17 +160,17 @@ This filter allows freqtrade to ignore pairs until they have been listed for at Offsets an incoming pairlist by a given `offset` value. -As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split -a larger pairlist on two bot instances. +As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split a larger pairlist on two bot instances. -Example to remove the first 10 pairs from the pairlist: +Example to remove the first 10 pairs from the pairlist, and takes the next 20 (taking items 10-30 of the initial list): ```json "pairlists": [ // ... { "method": "OffsetFilter", - "offset": 10 + "offset": 10, + "number_assets": 20 } ], ``` diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index 573a573a6..e0f8414ef 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -19,6 +19,7 @@ class OffsetFilter(IPairList): super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) self._offset = pairlistconfig.get('offset', 0) + self._number_pairs = pairlistconfig.get('number_assets', 0) if self._offset < 0: raise OperationalException("OffsetFilter requires offset to be >= 0") @@ -36,7 +37,9 @@ class OffsetFilter(IPairList): """ Short whitelist method description - used for startup-messages """ - return f"{self.name} - Offseting pairs by {self._offset}." + if self._number_pairs: + return f"{self.name} - Taking {self._number_pairs} Pairs, starting from {self._offset}." + return f"{self.name} - Offsetting pairs by {self._offset}." def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ @@ -50,5 +53,9 @@ class OffsetFilter(IPairList): self.log_once(f"Offset of {self._offset} is larger than " + f"pair count of {len(pairlist)}", logger.warning) pairs = pairlist[self._offset:] + if self._number_pairs: + pairs = pairs[:self._number_pairs] + self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info) + return pairs diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d80f23c8a..c29e619b1 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -470,12 +470,16 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "BTC", ['ETH/BTC', 'TKN/BTC']), # VolumePairList with no offset = unchanged pairlist ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, - {"method": "OffsetFilter", "offset": 0}], + {"method": "OffsetFilter", "offset": 0, "number_assets": 0}], "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']), # VolumePairList with offset = 2 ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, {"method": "OffsetFilter", "offset": 2}], "USDT", ['ADAHALF/USDT', 'ADADOUBLE/USDT']), + # VolumePairList with offset and limit + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 1, "number_assets": 2}], + "USDT", ['NANO/USDT', 'ADAHALF/USDT']), # VolumePairList with higher offset, than total pairlist ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, {"method": "OffsetFilter", "offset": 100}], @@ -1152,6 +1156,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo "0.01 and above 0.99 over the last days.'}]", None ), + ({"method": "OffsetFilter", "offset": 5, "number_assets": 10}, + "[{'OffsetFilter': 'OffsetFilter - Taking 10 Pairs, starting from 5.'}]", + None + ), ]) def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, desc_expected, exception_expected): From 5767d652bf9ffb97b4498c95cff9fc0e5c1654cf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 13:52:58 +0200 Subject: [PATCH 11/53] Add explicit test and document behavior --- freqtrade/exchange/exchange.py | 3 ++- freqtrade/exchange/okx.py | 8 +++++--- tests/exchange/test_okx.py | 27 +++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d2a01f394..86f80871b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -630,7 +630,8 @@ class Exchange: if required_candle_call_count > 5: # Only allow 5 calls per pair to somewhat limit the impact raise OperationalException( - f"This strategy requires {startup_candles} candles to start, which is more than 5x " + f"This strategy requires {startup_candles} candles to start, " + "which is more than 5x " f"the amount of candles {self.name} provides for {timeframe}.") elif required_candle_call_count > 1: raise OperationalException( diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 6d25bb12b..c8324e62e 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -41,11 +41,13 @@ class Okx(Exchange): net_only = True def ohlcv_candle_limit( - self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: """ Exchange ohlcv candle limit - Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits - per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit + OKX has the following behaviour: + * 300 candles for uptodate data + * 100 candles for historic data + * 100 candles for additional candles (not futures or spot). :param timeframe: Timeframe to check :param candle_type: Candle-type :param since_ms: Candle-type diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index f6bdd35ad..2804d471a 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -1,12 +1,39 @@ +from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, PropertyMock import pytest from freqtrade.enums import MarginMode, TradingMode +from freqtrade.enums.candletype import CandleType +from freqtrade.exchange.exchange import timeframe_to_minutes from tests.conftest import get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers +def test_okx_ohlcv_candle_limit(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf, id='okx') + timeframes = ('1m', '5m', '1h') + start_time = int(datetime(2021, 1, 1, tzinfo=timezone.utc).timestamp() * 1000) + + for timeframe in timeframes: + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT) == 300 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES) == 300 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.MARK) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUNDING_RATE) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, start_time) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, start_time) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.MARK, start_time) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUNDING_RATE, start_time) == 100 + one_call = int((datetime.now(timezone.utc) - timedelta( + minutes=290 * timeframe_to_minutes(timeframe))).timestamp() * 1000) + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, one_call) == 300 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, one_call) == 300 + one_call = int((datetime.now(timezone.utc) - timedelta( + minutes=320 * timeframe_to_minutes(timeframe))).timestamp() * 1000) + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, one_call) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, one_call) == 100 + + def test_get_maintenance_ratio_and_amt_okx( default_conf, mocker, From 1c20fb7638d433e0ba703ff995c572a42ecb65ac Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 14 May 2022 16:37:04 +0300 Subject: [PATCH 12/53] Refresh open_rate and stoploss on order replacement. --- freqtrade/optimize/backtesting.py | 4 ++++ freqtrade/persistence/trade_model.py | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 45300b744..f439e4e63 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -780,6 +780,8 @@ class Backtesting: # interest_rate=interest_rate, orders=[], ) + elif trade.nr_of_successful_entries == 0: + trade.open_rate = propose_rate trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) @@ -940,6 +942,8 @@ class Backtesting: requested_rate=requested_rate, requested_stake=(order.remaining * order.price), direction='short' if trade.is_short else 'long') + trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, + initial=False, refresh=True) else: # assumption: there can't be multiple open entry orders at any given time return (trade.nr_of_successful_entries == 0) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index b1fff5cf3..28cca5532 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -491,7 +491,7 @@ class LocalTrade(): self.stoploss_last_update = datetime.utcnow() def adjust_stop_loss(self, current_price: float, stoploss: float, - initial: bool = False) -> None: + initial: bool = False, refresh: bool = False) -> None: """ This adjusts the stop loss to it's most recently observed setting :param current_price: Current rate the asset is traded @@ -516,8 +516,7 @@ class LocalTrade(): new_loss = max(self.liquidation_price, new_loss) # no stop loss assigned yet - if self.initial_stop_loss_pct is None: - logger.debug(f"{self.pair} - Assigning new stoploss...") + if self.initial_stop_loss_pct is None or refresh: self._set_stop_loss(new_loss, stoploss) self.initial_stop_loss = new_loss self.initial_stop_loss_pct = -1 * abs(stoploss) From ec54b47b6e36394f5f633af0f7d5a5cdc960ccfb Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 14 May 2022 16:39:27 +0300 Subject: [PATCH 13/53] Flake fix. --- tests/optimize/test_backtest_detail.py | 35 ++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 0441d4214..4b4c446e0 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -762,7 +762,7 @@ tc48 = BTContainer(data=[ [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust [3, 5100, 5100, 4650, 4750, 6172, 0, 1], [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], - stop_loss=-0.01, roi={"0": 0.10}, profit_perc=-0.087, + stop_loss=-0.2, roi={"0": 0.10}, profit_perc=-0.087, use_exit_signal=True, timeout=1000, custom_entry_price=4200, adjust_entry_price=5200, trades=[BTrade(exit_reason=ExitType.EXIT_SIGNAL, open_tick=1, close_tick=4, is_short=False)] @@ -777,7 +777,7 @@ tc49 = BTContainer(data=[ [2, 4900, 5250, 4900, 5100, 6172, 0, 0, 0, 0], # Order readjust [3, 5100, 5100, 4650, 4750, 6172, 0, 0, 0, 1], [4, 4750, 4950, 4350, 4750, 6172, 0, 0, 0, 0]], - stop_loss=-0.01, roi={"0": 0.10}, profit_perc=0.05, + stop_loss=-0.2, roi={"0": 0.10}, profit_perc=0.05, use_exit_signal=True, timeout=1000, custom_entry_price=5300, adjust_entry_price=5000, trades=[BTrade(exit_reason=ExitType.EXIT_SIGNAL, open_tick=1, close_tick=4, is_short=True)] @@ -811,6 +811,35 @@ tc51 = BTContainer(data=[ trades=[] ) +# Test 52: Custom-entry-price below all candles - readjust order - stoploss +tc52 = BTContainer(data=[ + # D O H L C V EL XL ES Xs BT + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], # stoploss hit? + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.03, roi={"0": 0.10}, profit_perc=-0.03, + use_exit_signal=True, timeout=1000, + custom_entry_price=4200, adjust_entry_price=5200, + trades=[BTrade(exit_reason=ExitType.STOP_LOSS, open_tick=1, close_tick=2, is_short=False)] +) + + +# Test 53: Custom-entry-price short above all candles - readjust order - stoploss +tc53 = BTContainer(data=[ + # D O H L C V EL XL ES Xs BT + [0, 5000, 5050, 4950, 5000, 6172, 0, 0, 1, 0], + [1, 5000, 5200, 4951, 5000, 6172, 0, 0, 0, 0], # enter trade (signal on last candle) + [2, 4900, 5250, 4900, 5100, 6172, 0, 0, 0, 0], # Order readjust + [3, 5100, 5100, 4650, 4750, 6172, 0, 0, 0, 1], # stoploss hit? + [4, 4750, 4950, 4350, 4750, 6172, 0, 0, 0, 0]], + stop_loss=-0.03, roi={"0": 0.10}, profit_perc=-0.03, + use_exit_signal=True, timeout=1000, + custom_entry_price=5300, adjust_entry_price=5000, + trades=[BTrade(exit_reason=ExitType.STOP_LOSS, open_tick=1, close_tick=2, is_short=True)] +) + TESTS = [ tc0, tc1, @@ -864,6 +893,8 @@ TESTS = [ tc49, tc50, tc51, + tc52, + tc53, ] From c27e0a0a1b8837525fe93cfaa459a78c4dce8f2b Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 14 May 2022 16:56:56 +0300 Subject: [PATCH 14/53] Allow SL refresh only if no filled entry orders. --- freqtrade/persistence/trade_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 28cca5532..bbdeeef47 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -502,6 +502,7 @@ class LocalTrade(): if initial and not (self.stop_loss is None or self.stop_loss == 0): # Don't modify if called with initial and nothing to do return + refresh = False if self.nr_of_successful_entries > 0 else refresh leverage = self.leverage or 1.0 if self.is_short: From 3b144392407501a7df9ecb32c04db6c434dbbcb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 16:16:32 +0200 Subject: [PATCH 15/53] Slightly improve performance of order adjusts Avoind 2nd call to `get_rate()`. closes #6821 --- freqtrade/freqtradebot.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 03cd322a6..07b055309 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -536,7 +536,8 @@ class FreqtradeBot(LoggingMixin): if stake_amount is not None and stake_amount > 0.0: # We should increase our position - self.execute_entry(trade.pair, stake_amount, trade=trade, is_short=trade.is_short) + self.execute_entry(trade.pair, stake_amount, price=current_rate, + trade=trade, is_short=trade.is_short) if stake_amount is not None and stake_amount < 0.0: # We should decrease our position @@ -586,6 +587,7 @@ class FreqtradeBot(LoggingMixin): ordertype: Optional[str] = None, enter_tag: Optional[str] = None, trade: Optional[Trade] = None, + order_adjust: bool = False ) -> bool: """ Executes a limit buy for the given pair @@ -601,7 +603,7 @@ class FreqtradeBot(LoggingMixin): pos_adjust = trade is not None enter_limit_requested, stake_amount, leverage = self.get_valid_enter_price_and_stake( - pair, price, stake_amount, trade_side, enter_tag, trade) + pair, price, stake_amount, trade_side, enter_tag, trade, order_adjust) if not stake_amount: return False @@ -746,23 +748,26 @@ class FreqtradeBot(LoggingMixin): self, pair: str, price: Optional[float], stake_amount: float, trade_side: LongShort, entry_tag: Optional[str], - trade: Optional[Trade] + trade: Optional[Trade], + order_adjust: bool, ) -> Tuple[float, float, float]: if price: enter_limit_requested = price else: # Calculate price - proposed_enter_rate = self.exchange.get_rate( + enter_limit_requested = self.exchange.get_rate( pair, side='entry', is_short=(trade_side == 'short'), refresh=True) + if not order_adjust: + # Don't call custom_entry_price in order-adjust scenario custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, - default_retval=proposed_enter_rate)( + default_retval=enter_limit_requested)( pair=pair, current_time=datetime.now(timezone.utc), - proposed_rate=proposed_enter_rate, entry_tag=entry_tag, + proposed_rate=enter_limit_requested, entry_tag=entry_tag, side=trade_side, ) - enter_limit_requested = self.get_valid_price(custom_entry_price, proposed_enter_rate) + enter_limit_requested = self.get_valid_price(custom_entry_price, enter_limit_requested) if not enter_limit_requested: raise PricingError('Could not determine entry price.') @@ -1212,7 +1217,8 @@ class FreqtradeBot(LoggingMixin): stake_amount=(order_obj.remaining * order_obj.price), price=adjusted_entry_price, trade=trade, - is_short=trade.is_short + is_short=trade.is_short, + order_adjust=True, ) def cancel_all_open_orders(self) -> None: From a947a1316b005c29b8a5ae58665f6f52ee884bb1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 17:42:01 +0200 Subject: [PATCH 16/53] Add test to ensure stoploss is set properly in live --- tests/test_integration.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 020f77fed..d2ad8c981 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -372,11 +372,15 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, fee, mocker) -> None: freqtrade.enter_positions() assert len(Trade.get_trades().all()) == 1 - trade = Trade.get_trades().first() + trade: Trade = Trade.get_trades().first() assert len(trade.orders) == 1 assert trade.open_order_id is not None assert pytest.approx(trade.stake_amount) == 60 assert trade.open_rate == 1.96 + assert trade.stop_loss_pct is None + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 + assert trade.initial_stop_loss_pct is None # No adjustment freqtrade.process() trade = Trade.get_trades().first() @@ -392,6 +396,10 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, fee, mocker) -> None: assert trade.open_order_id is not None # Open rate is not adjusted yet assert trade.open_rate == 1.96 + assert trade.stop_loss_pct is None + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 + assert trade.initial_stop_loss_pct is None # Fill order mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=True) @@ -401,6 +409,10 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, fee, mocker) -> None: assert trade.open_order_id is None # Open rate is not adjusted yet assert trade.open_rate == 1.99 + assert trade.stop_loss_pct == -0.1 + assert trade.stop_loss == 1.99 * 0.9 + assert trade.initial_stop_loss == 1.99 * 0.9 + assert trade.initial_stop_loss_pct == -0.1 # 2nd order - not filling freqtrade.strategy.adjust_trade_position = MagicMock(return_value=120) From 116b58e97cad2b86aff5e20d97f494b5ef9abd41 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 19:30:42 +0200 Subject: [PATCH 17/53] add "date_minus_candles" method --- freqtrade/exchange/exchange.py | 24 +++++++++++++++++++++--- tests/exchange/test_exchange.py | 17 ++++++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 86f80871b..560da8eb2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2687,9 +2687,10 @@ def timeframe_to_msecs(timeframe: str) -> int: def timeframe_to_prev_date(timeframe: str, date: datetime = None) -> datetime: """ - Use Timeframe and determine last possible candle. + Use Timeframe and determine the candle start date for this date. + Does not round when given a candle start date. :param timeframe: timeframe in string format (e.g. "5m") - :param date: date to use. Defaults to utcnow() + :param date: date to use. Defaults to now(utc) :returns: date of previous candle (with utc timezone) """ if not date: @@ -2704,7 +2705,7 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: """ Use Timeframe and determine next candle. :param timeframe: timeframe in string format (e.g. "5m") - :param date: date to use. Defaults to utcnow() + :param date: date to use. Defaults to now(utc) :returns: date of next candle (with utc timezone) """ if not date: @@ -2714,6 +2715,23 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) +def date_minus_candles( + timeframe: str, candle_count: int, date: Optional[datetime] = None) -> datetime: + """ + subtract X candles from a date. + :param timeframe: timeframe in string format (e.g. "5m") + :param candle_count: Amount of candles to subtract. + :param date: date to use. Defaults to now(utc) + + """ + if not date: + date = datetime.now(timezone.utc) + + tf_min = timeframe_to_minutes(timeframe) + new_date = timeframe_to_prev_date(timeframe, date) - timedelta(minutes=tf_min * candle_count) + return new_date + + def market_is_active(market: Dict) -> bool: """ Return True if the market is active. diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 9d7b77a8e..9dd4e6342 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -17,9 +17,9 @@ from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOr from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff, remove_credentials) -from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs, - timeframe_to_next_date, timeframe_to_prev_date, - timeframe_to_seconds) +from freqtrade.exchange.exchange import (date_minus_candles, market_is_active, timeframe_to_minutes, + timeframe_to_msecs, timeframe_to_next_date, + timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re @@ -3431,6 +3431,17 @@ def test_timeframe_to_next_date(): assert timeframe_to_next_date("5m", date) == date + timedelta(minutes=5) +def test_date_minus_candles(): + + date = datetime(2019, 8, 12, 13, 25, 0, tzinfo=timezone.utc) + + assert date_minus_candles("5m", 3, date) == date - timedelta(minutes=15) + assert date_minus_candles("5m", 5, date) == date - timedelta(minutes=25) + assert date_minus_candles("1m", 6, date) == date - timedelta(minutes=6) + assert date_minus_candles("1h", 3, date) == date - timedelta(hours=3, minutes=25) + assert date_minus_candles("1h", 3) == timeframe_to_prev_date('1h') - timedelta(hours=3) + + @pytest.mark.parametrize( "market_symbol,base,quote,exchange,spot,margin,futures,trademode,add_dict,expected_result", [ From d60d0f64d209d1013e2d32938f72bc8a94598af8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 19:32:28 +0200 Subject: [PATCH 18/53] Revert ohlcv_candle_limit logic for okx --- freqtrade/exchange/exchange.py | 5 ++++- freqtrade/exchange/okx.py | 18 ++++++++---------- tests/exchange/test_okx.py | 3 +++ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 560da8eb2..57a7f2086 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -619,7 +619,10 @@ class Exchange: Checks if required startup_candles is more than ohlcv_candle_limit(). Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. """ - candle_limit = self.ohlcv_candle_limit(timeframe, self._config['candle_type_def'], None) + + candle_limit = self.ohlcv_candle_limit( + timeframe, self._config['candle_type_def'], + date_minus_candles(timeframe, startup_candles)) # Require one more candle - to account for the still open candle. candle_count = startup_candles + 1 # Allow 5 calls to the exchange per pair diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index c8324e62e..ad41984e7 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -10,7 +10,7 @@ from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier -from freqtrade.exchange.exchange import timeframe_to_minutes +from freqtrade.exchange.exchange import date_minus_candles logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ class Okx(Exchange): """ _ft_has: Dict = { - "ohlcv_candle_limit": 300, # Warning, special case with data prior to X months + "ohlcv_candle_limit": 100, # Warning, special case with data prior to X months "mark_ohlcv_timeframe": "4h", "funding_fee_timeframe": "8h", } @@ -53,15 +53,13 @@ class Okx(Exchange): :param since_ms: Candle-type :return: Candle limit as integer """ - now = datetime.now(timezone.utc) - offset_mins = timeframe_to_minutes(timeframe) * self._ft_has['ohlcv_candle_limit'] - if since_ms and since_ms < ((now - timedelta(minutes=offset_mins)).timestamp() * 1000): - return 100 - if candle_type not in (CandleType.FUTURES, CandleType.SPOT): - return 100 + if ( + candle_type in (CandleType.FUTURES, CandleType.SPOT) and + (not since_ms or since_ms > (date_minus_candles(timeframe, 300).timestamp() * 1000)) + ): + return 300 - return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( - timeframe, self._ft_has.get('ohlcv_candle_limit'))) + return super().ohlcv_candle_limit(timeframe, candle_type, since_ms) @retrier def additional_exchange_init(self) -> None: diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 2804d471a..19c09ad9e 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -20,14 +20,17 @@ def test_okx_ohlcv_candle_limit(default_conf, mocker): assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES) == 300 assert exchange.ohlcv_candle_limit(timeframe, CandleType.MARK) == 100 assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUNDING_RATE) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, start_time) == 100 assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, start_time) == 100 assert exchange.ohlcv_candle_limit(timeframe, CandleType.MARK, start_time) == 100 assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUNDING_RATE, start_time) == 100 one_call = int((datetime.now(timezone.utc) - timedelta( minutes=290 * timeframe_to_minutes(timeframe))).timestamp() * 1000) + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, one_call) == 300 assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, one_call) == 300 + one_call = int((datetime.now(timezone.utc) - timedelta( minutes=320 * timeframe_to_minutes(timeframe))).timestamp() * 1000) assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, one_call) == 100 From 9143e9ecb15c9756b6e0f4a7f437a15cddc12385 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 15:12:29 +0200 Subject: [PATCH 19/53] Add some safety measures for new startup_candles verification --- freqtrade/exchange/exchange.py | 3 ++- freqtrade/exchange/okx.py | 1 - tests/exchange/test_exchange.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 57a7f2086..a07ea3596 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -622,7 +622,8 @@ class Exchange: candle_limit = self.ohlcv_candle_limit( timeframe, self._config['candle_type_def'], - date_minus_candles(timeframe, startup_candles)) + int(date_minus_candles(timeframe, startup_candles).timestamp() * 1000) + if timeframe else None) # Require one more candle - to account for the still open candle. candle_count = startup_candles + 1 # Allow 5 calls to the exchange per pair diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index ad41984e7..c0431c7fc 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional, Tuple import ccxt diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 9dd4e6342..e580c82d3 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -939,6 +939,7 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker): def test_validate_timeframes_not_in_config(default_conf, mocker): + # TODO: this test does not assert ... del default_conf["timeframe"] api_mock = MagicMock() id_mock = PropertyMock(return_value='test_exchange') @@ -954,6 +955,7 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') mocker.patch('freqtrade.exchange.Exchange.validate_pricing') + mocker.patch('freqtrade.exchange.Exchange.validate_required_startup_candles') Exchange(default_conf) From 18fd3bb3332f2d50401fa6b428d073615b02db7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 15:30:57 +0200 Subject: [PATCH 20/53] Update stoploss handling for entry-order adjustment --- freqtrade/optimize/backtesting.py | 6 +----- freqtrade/persistence/trade_model.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f439e4e63..621812b0a 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -780,8 +780,6 @@ class Backtesting: # interest_rate=interest_rate, orders=[], ) - elif trade.nr_of_successful_entries == 0: - trade.open_rate = propose_rate trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) @@ -814,11 +812,11 @@ class Backtesting: remaining=amount, cost=stake_amount + trade.fee_open, ) + trade.orders.append(order) if pos_adjust and self._get_order_filled(order.price, row): order.close_bt_order(current_time, trade) else: trade.open_order_id = str(self.order_id_counter) - trade.orders.append(order) trade.recalc_trade_from_orders() return trade @@ -942,8 +940,6 @@ class Backtesting: requested_rate=requested_rate, requested_stake=(order.remaining * order.price), direction='short' if trade.is_short else 'long') - trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, - initial=False, refresh=True) else: # assumption: there can't be multiple open entry orders at any given time return (trade.nr_of_successful_entries == 0) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index bbdeeef47..358e776e3 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -153,6 +153,7 @@ class Order(_DECL_BASE): and len(trade.select_filled_orders(trade.entry_side)) == 1): trade.open_rate = self.price trade.recalc_open_trade_value() + trade.adjust_stop_loss(trade.open_rate, trade.stop_loss_pct, refresh=True) @staticmethod def update_orders(orders: List['Order'], order: Dict[str, Any]): @@ -502,7 +503,7 @@ class LocalTrade(): if initial and not (self.stop_loss is None or self.stop_loss == 0): # Don't modify if called with initial and nothing to do return - refresh = False if self.nr_of_successful_entries > 0 else refresh + refresh = True if refresh and self.nr_of_successful_entries == 1 else False leverage = self.leverage or 1.0 if self.is_short: From 706994340f36e5336e35afe4a580015992e131b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 17:06:40 +0200 Subject: [PATCH 21/53] Fix bad docstring --- freqtrade/exchange/exchange.py | 2 +- freqtrade/exchange/okx.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a07ea3596..ee804aa68 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -317,7 +317,7 @@ class Exchange: per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit :param timeframe: Timeframe to check :param candle_type: Candle-type - :param since_ms: Candle-type + :param since_ms: Starting timestamp :return: Candle limit as integer """ return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index c0431c7fc..012f51080 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -49,7 +49,7 @@ class Okx(Exchange): * 100 candles for additional candles (not futures or spot). :param timeframe: Timeframe to check :param candle_type: Candle-type - :param since_ms: Candle-type + :param since_ms: Starting timestamp :return: Candle limit as integer """ if ( From a8f064a8cb4b84914820c98bc895ff6c7a0dda71 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 17:33:00 +0200 Subject: [PATCH 22/53] Fix exit_reason assignment in live mode --- freqtrade/freqtradebot.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 07b055309..315db3ae6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1414,6 +1414,7 @@ class FreqtradeBot(LoggingMixin): open_date=trade.open_date_utc, ) exit_type = 'exit' + exit_reason = exit_tag or exit_check.exit_reason if exit_check.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS): exit_type = 'stoploss' @@ -1431,7 +1432,7 @@ class FreqtradeBot(LoggingMixin): pair=trade.pair, trade=trade, current_time=datetime.now(timezone.utc), proposed_rate=proposed_limit_rate, current_profit=current_profit, - exit_tag=exit_check.exit_reason) + exit_tag=exit_reason) limit = self.get_valid_price(custom_exit_price, proposed_limit_rate) @@ -1448,8 +1449,8 @@ class FreqtradeBot(LoggingMixin): if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, - time_in_force=time_in_force, exit_reason=exit_check.exit_reason, - sell_reason=exit_check.exit_reason, # sellreason -> compatibility + time_in_force=time_in_force, exit_reason=exit_reason, + sell_reason=exit_reason, # sellreason -> compatibility current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of exiting {trade.pair}") return False @@ -1478,7 +1479,7 @@ class FreqtradeBot(LoggingMixin): trade.open_order_id = order['id'] trade.exit_order_status = '' trade.close_rate_requested = limit - trade.exit_reason = exit_tag or exit_check.exit_reason + trade.exit_reason = exit_reason # Lock pair for one candle to prevent immediate re-trading self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), From a0b25938f472d4941b7f08986a88c0c69b356e7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 17:41:50 +0200 Subject: [PATCH 23/53] Fix exit_reason assignment in backtesting --- freqtrade/optimize/backtesting.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 621812b0a..64107ae18 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -535,6 +535,7 @@ class Backtesting: if exit_.exit_flag: trade.close_date = exit_candle_time + exit_reason = exit_.exit_reason trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) try: @@ -545,6 +546,15 @@ class Backtesting: current_profit = trade.calc_profit_ratio(closerate) order_type = self.strategy.order_types['exit'] if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT): + # Checks and adds an exit tag, after checking that the length of the + # row has the length for an exit tag column + if( + len(row) > EXIT_TAG_IDX + and row[EXIT_TAG_IDX] is not None + and len(row[EXIT_TAG_IDX]) > 0 + and exit_.exit_type in (ExitType.EXIT_SIGNAL,) + ): + exit_reason = row[EXIT_TAG_IDX] # Custom exit pricing only for exit-signals if order_type == 'limit': closerate = strategy_safe_wrapper(self.strategy.custom_exit_price, @@ -552,7 +562,7 @@ class Backtesting: pair=trade.pair, trade=trade, current_time=exit_candle_time, proposed_rate=closerate, current_profit=current_profit, - exit_tag=exit_.exit_reason) + exit_tag=exit_reason) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately if trade.is_short: @@ -566,22 +576,12 @@ class Backtesting: pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount, rate=closerate, time_in_force=time_in_force, - sell_reason=exit_.exit_reason, # deprecated - exit_reason=exit_.exit_reason, + sell_reason=exit_reason, # deprecated + exit_reason=exit_reason, current_time=exit_candle_time): return None - trade.exit_reason = exit_.exit_reason - - # Checks and adds an exit tag, after checking that the length of the - # row has the length for an exit tag column - if( - len(row) > EXIT_TAG_IDX - and row[EXIT_TAG_IDX] is not None - and len(row[EXIT_TAG_IDX]) > 0 - and exit_.exit_type in (ExitType.EXIT_SIGNAL,) - ): - trade.exit_reason = row[EXIT_TAG_IDX] + trade.exit_reason = exit_reason self.order_id_counter += 1 order = Order( From 86af3fe0e7766a3dd86701d8142aeb327bafd7d2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 19:22:12 +0200 Subject: [PATCH 24/53] Update image versions from 3.9 to 3.10 --- .github/workflows/ci.yml | 6 +++--- Dockerfile | 2 +- docker/Dockerfile.armhf | 2 +- setup.cfg | 2 +- setup.sh | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96575f034..09946e6b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -273,7 +273,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: 3.10 - name: pre-commit dependencies run: | @@ -292,7 +292,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: 3.10 - name: Documentation build run: | @@ -358,7 +358,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.9 - name: Extract branch name shell: bash diff --git a/Dockerfile b/Dockerfile index 8f5b85698..5f7b52265 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.9-slim-bullseye as base +FROM python:3.10.4-slim-bullseye as base # Setup env ENV LANG C.UTF-8 diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index 16f2aebcd..73fc681eb 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM python:3.9.9-slim-bullseye as base +FROM python:3.9.12-slim-bullseye as base # Setup env ENV LANG C.UTF-8 diff --git a/setup.cfg b/setup.cfg index edbd320c3..042517ec9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ tests_require = pytest-mock packages = find: -python_requires = >=3.6 +python_requires = >=3.8 [options.entry_points] console_scripts = diff --git a/setup.sh b/setup.sh index dcf6c02c7..bb51c3a2f 100755 --- a/setup.sh +++ b/setup.sh @@ -25,7 +25,7 @@ function check_installed_python() { exit 2 fi - for v in 9 10 8 + for v in 10 9 8 do PYTHON="python3.${v}" which $PYTHON From 008ee148890b3c396c311483340bfb698ebd925a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 19:25:27 +0200 Subject: [PATCH 25/53] Improve ci to run on ubuntu 22.04 --- .github/workflows/ci.yml | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09946e6b5..d11285ba4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-18.04, ubuntu-20.04 ] + os: [ ubuntu-18.04, ubuntu-20.04, ubuntu-22.04 ] python-version: ["3.8", "3.9", "3.10"] steps: @@ -70,7 +70,7 @@ jobs: if: matrix.python-version == '3.9' - name: Coveralls - if: (runner.os == 'Linux' && matrix.python-version == '3.8') + if: (runner.os == 'Linux' && matrix.python-version == '3.9') env: # Coveralls token. Not used as secret due to github not providing secrets to forked repositories COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu @@ -157,24 +157,9 @@ jobs: pip install -e . - name: Tests - if: (runner.os != 'Linux' || matrix.python-version != '3.8') run: | pytest --random-order - - name: Tests (with cov) - if: (runner.os == 'Linux' && matrix.python-version == '3.8') - run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc - - - name: Coveralls - if: (runner.os == 'Linux' && matrix.python-version == '3.8') - env: - # Coveralls token. Not used as secret due to github not providing secrets to forked repositories - COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu - run: | - # Allow failure for coveralls - coveralls -v || true - - name: Backtesting run: | cp config_examples/config_bittrex.example.json config.json @@ -273,7 +258,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.10 + python-version: "3.10" - name: pre-commit dependencies run: | @@ -292,7 +277,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.10 + python-version: "3.10" - name: Documentation build run: | @@ -358,7 +343,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: "3.9" - name: Extract branch name shell: bash From e21f6a7787091e2c3831f872bf26aaed3df0184c Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 16 May 2022 07:28:40 +0900 Subject: [PATCH 26/53] missing newline --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 1a9be4503..259d2c831 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1417,7 +1417,7 @@ class Telegram(RPCHandler): "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" "*/forceexit |all:* `Instantly exits the given trade or all trades, " "regardless of profit`\n" - "*/fe |all:* `Alias to /forceexit`" + "*/fe |all:* `Alias to /forceexit`\n" f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}" "*/delete :* `Instantly delete the given trade in the database`\n" "*/whitelist:* `Show current whitelist` \n" From 2cb8eecf18eed273df41ee2702e54def89831b03 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 16 May 2022 07:43:36 +0900 Subject: [PATCH 27/53] add space --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 259d2c831..f26de8b5c 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1410,7 +1410,7 @@ class Telegram(RPCHandler): "Optionally takes a rate at which to sell " "(only applies to limit orders).` \n") message = ( - "_BotControl_\n" + "_Bot Control_\n" "------------\n" "*/start:* `Starts the trader`\n" "*/stop:* Stops the trader\n" From 4fc6857d8778cb423ab228750a8edb23c2cfe0c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:25 +0000 Subject: [PATCH 28/53] Bump time-machine from 2.6.0 to 2.7.0 Bumps [time-machine](https://github.com/adamchainz/time-machine) from 2.6.0 to 2.7.0. - [Release notes](https://github.com/adamchainz/time-machine/releases) - [Changelog](https://github.com/adamchainz/time-machine/blob/main/HISTORY.rst) - [Commits](https://github.com/adamchainz/time-machine/compare/2.6.0...2.7.0) --- updated-dependencies: - dependency-name: time-machine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 60f4da1a7..58eaef3e2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,7 +16,7 @@ pytest-mock==3.7.0 pytest-random-order==1.0.4 isort==5.10.1 # For datetime mocking -time-machine==2.6.0 +time-machine==2.7.0 # Convert jupyter notebooks to markdown documents nbconvert==6.5.0 From 47c116a423cabe9a0444c29be9edef465af86ecb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:28 +0000 Subject: [PATCH 29/53] Bump fastapi from 0.76.0 to 0.78.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.76.0 to 0.78.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.76.0...0.78.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bed3a7fed..14edbe16c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ orjson==3.6.8 sdnotify==0.3.2 # API Server -fastapi==0.76.0 +fastapi==0.78.0 uvicorn==0.17.6 pyjwt==2.3.0 aiofiles==0.8.0 From 748055892cbc0f32a6610960ec0e58521e6b45fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:33 +0000 Subject: [PATCH 30/53] Bump plotly from 5.7.0 to 5.8.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.7.0 to 5.8.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.7.0...v5.8.0) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index d9faed301..e17efbc71 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.7.0 +plotly==5.8.0 From 9fc21686ede5c7f28f978a9198535307a2a92b14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:40 +0000 Subject: [PATCH 31/53] Bump flake8-tidy-imports from 4.7.0 to 4.8.0 Bumps [flake8-tidy-imports](https://github.com/adamchainz/flake8-tidy-imports) from 4.7.0 to 4.8.0. - [Release notes](https://github.com/adamchainz/flake8-tidy-imports/releases) - [Changelog](https://github.com/adamchainz/flake8-tidy-imports/blob/main/HISTORY.rst) - [Commits](https://github.com/adamchainz/flake8-tidy-imports/compare/4.7.0...4.8.0) --- updated-dependencies: - dependency-name: flake8-tidy-imports dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 60f4da1a7..17a505912 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ coveralls==3.3.1 flake8==4.0.1 -flake8-tidy-imports==4.7.0 +flake8-tidy-imports==4.8.0 mypy==0.950 pre-commit==2.19.0 pytest==7.1.2 From 9e44d697746369fa6467e053698a698365069cfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:53 +0000 Subject: [PATCH 32/53] Bump ccxt from 1.81.81 to 1.82.61 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.81.81 to 1.82.61. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.81.81...1.82.61) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bed3a7fed..cdd72403d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.3 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.81.81 +ccxt==1.82.61 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From a8b4066f85fdb08c254e06be91a3818e62dc1855 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:57 +0000 Subject: [PATCH 33/53] Bump mkdocs-material from 8.2.14 to 8.2.15 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.2.14 to 8.2.15. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.2.14...8.2.15) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index b26e448ea..3fa35d80d 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.2.14 +mkdocs-material==8.2.15 mdx_truly_sane_lists==1.2 pymdown-extensions==9.4 jinja2==3.1.2 From dd1b84f938db950821d4d40e14be83e1b0faff5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:02:00 +0000 Subject: [PATCH 34/53] Bump filelock from 3.6.0 to 3.7.0 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.6.0 to 3.7.0. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/py-filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.6.0...3.7.0) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 32fc3f4b9..17a7c7b8c 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,5 +5,5 @@ scipy==1.8.0 scikit-learn==1.0.2 scikit-optimize==0.9.0 -filelock==3.6.0 +filelock==3.7.0 progressbar2==4.0.0 From bd65236e17f53ed19abf6a687ff0cd8d856d838f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 04:37:25 +0000 Subject: [PATCH 35/53] Bump pyjwt from 2.3.0 to 2.4.0 Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.3.0...2.4.0) --- updated-dependencies: - dependency-name: pyjwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 95826b71c..90ddcd1b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ sdnotify==0.3.2 # API Server fastapi==0.78.0 uvicorn==0.17.6 -pyjwt==2.3.0 +pyjwt==2.4.0 aiofiles==0.8.0 psutil==5.9.0 From f5183df0f1c6e41fb3b8d39fca18b8f829bf73d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 04:37:41 +0000 Subject: [PATCH 36/53] Bump scikit-learn from 1.0.2 to 1.1.0 Bumps [scikit-learn](https://github.com/scikit-learn/scikit-learn) from 1.0.2 to 1.1.0. - [Release notes](https://github.com/scikit-learn/scikit-learn/releases) - [Commits](https://github.com/scikit-learn/scikit-learn/compare/1.0.2...1.1.0) --- updated-dependencies: - dependency-name: scikit-learn dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 17a7c7b8c..0b91636f1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -3,7 +3,7 @@ # Required for hyperopt scipy==1.8.0 -scikit-learn==1.0.2 +scikit-learn==1.1.0 scikit-optimize==0.9.0 filelock==3.7.0 progressbar2==4.0.0 From 528509f809b2474218e24ac5442d889b4ca1fce9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 May 2022 19:18:13 +0200 Subject: [PATCH 37/53] Extract get_price_side from get_rate --- freqtrade/exchange/exchange.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ee804aa68..156216557 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1457,6 +1457,23 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def _get_price_side(self, side: str, is_short: bool, conf_strategy: Dict) -> str: + price_side = conf_strategy['price_side'] + + if price_side in ('same', 'other'): + price_map = { + ('entry', 'long', 'same'): 'bid', + ('entry', 'long', 'other'): 'ask', + ('entry', 'short', 'same'): 'ask', + ('entry', 'short', 'other'): 'bid', + ('exit', 'long', 'same'): 'ask', + ('exit', 'long', 'other'): 'bid', + ('exit', 'short', 'same'): 'bid', + ('exit', 'short', 'other'): 'ask', + } + price_side = price_map[(side, 'short' if is_short else 'long', price_side)] + return price_side + def get_rate(self, pair: str, refresh: bool, side: EntryExit, is_short: bool) -> float: """ @@ -1483,20 +1500,7 @@ class Exchange: conf_strategy = self._config.get(strat_name, {}) - price_side = conf_strategy['price_side'] - - if price_side in ('same', 'other'): - price_map = { - ('entry', 'long', 'same'): 'bid', - ('entry', 'long', 'other'): 'ask', - ('entry', 'short', 'same'): 'ask', - ('entry', 'short', 'other'): 'bid', - ('exit', 'long', 'same'): 'ask', - ('exit', 'long', 'other'): 'bid', - ('exit', 'short', 'same'): 'bid', - ('exit', 'short', 'other'): 'ask', - } - price_side = price_map[(side, 'short' if is_short else 'long', price_side)] + price_side = self._get_price_side(side, is_short, conf_strategy) price_side_word = price_side.capitalize() From a793cf8f05975767432d6b12bd31307e21923eb6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Mar 2022 07:10:46 +0200 Subject: [PATCH 38/53] Use ccxt's "precise" to do precise math --- freqtrade/exchange/exchange.py | 10 ++++++---- tests/exchange/test_exchange.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 156216557..d17c84f5c 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -16,6 +16,7 @@ import arrow import ccxt import ccxt.async_support as ccxt_async from cachetools import TTLCache +from ccxt import Precise from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision) from pandas import DataFrame @@ -704,10 +705,11 @@ class Exchange: # counting_mode=self.precisionMode, # )) if self.precisionMode == TICK_SIZE: - precision = self.markets[pair]['precision']['price'] - missing = price % precision - if missing != 0: - price = round(price - missing + precision, 10) + precision = Precise(str(self.markets[pair]['precision']['price'])) + price_str = Precise(str(price)) + missing = price_str.mod(precision) + if not missing.equals(Precise("0")): + price = round(float(str(price_str.sub(missing).add(precision))), 14) else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e580c82d3..53e6cc3f3 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -305,6 +305,7 @@ def test_amount_to_precision( (234.53, 4, 0.5, 235.0), (0.891534, 4, 0.0001, 0.8916), (64968.89, 4, 0.01, 64968.89), + (0.000000003483, 4, 1e-12, 0.000000003483), ]) def test_price_to_precision(default_conf, mocker, price, precision_mode, precision, expected): From c8e0fc926d756f9cd5b5eff539653b2f9e332c06 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Apr 2022 12:00:41 +0200 Subject: [PATCH 39/53] Update to do Builtin Precise math --- freqtrade/exchange/exchange.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d17c84f5c..8bbbf6d4d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -707,9 +707,9 @@ class Exchange: if self.precisionMode == TICK_SIZE: precision = Precise(str(self.markets[pair]['precision']['price'])) price_str = Precise(str(price)) - missing = price_str.mod(precision) - if not missing.equals(Precise("0")): - price = round(float(str(price_str.sub(missing).add(precision))), 14) + missing = price_str % precision + if not missing == Precise("0"): + price = round(float(str(price_str - missing + precision)), 14) else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) From d09b462930adf105d3f6a074d1f2b9f3d58e3ab4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Apr 2022 19:46:55 +0200 Subject: [PATCH 40/53] Add rudimentary tests for Precise "builtin operator" workings --- tests/exchange/test_ccxt_precise.py | 75 +++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/exchange/test_ccxt_precise.py diff --git a/tests/exchange/test_ccxt_precise.py b/tests/exchange/test_ccxt_precise.py new file mode 100644 index 000000000..026adb4c1 --- /dev/null +++ b/tests/exchange/test_ccxt_precise.py @@ -0,0 +1,75 @@ +from ccxt import Precise + + +ws = Precise('-1.123e-6') +ws = Precise('-1.123e-6') +xs = Precise('0.00000002') +ys = Precise('69696900000') +zs = Precise('0') + + +def test_precise(): + assert ys * xs == '1393.938' + assert xs * ys == '1393.938' + + assert ys + xs == '69696900000.00000002' + assert xs + ys == '69696900000.00000002' + assert xs - ys == '-69696899999.99999998' + assert ys - xs == '69696899999.99999998' + assert xs / ys == '0' + assert ys / xs == '3484845000000000000' + + assert ws * xs == '-0.00000000000002246' + assert xs * ws == '-0.00000000000002246' + + assert ws + xs == '-0.000001103' + assert xs + ws == '-0.000001103' + + assert xs - ws == '0.000001143' + assert ws - xs == '-0.000001143' + + assert xs / ws == '-0.017809439002671415' + assert ws / xs == '-56.15' + + assert zs * ws == '0' + assert zs * xs == '0' + assert zs * ys == '0' + assert ws * zs == '0' + assert xs * zs == '0' + assert ys * zs == '0' + + assert zs + ws == '-0.000001123' + assert zs + xs == '0.00000002' + assert zs + ys == '69696900000' + assert ws + zs == '-0.000001123' + assert xs + zs == '0.00000002' + assert ys + zs == '69696900000' + + assert abs(Precise('-500.1')) == '500.1' + assert abs(Precise('213')) == '213' + + assert abs(Precise('-500.1')) == '500.1' + assert -Precise('213') == '-213' + + assert Precise('10.1') % Precise('0.5') == '0.1' + assert Precise('5550') % Precise('120') == '30' + + assert Precise('-0.0') == Precise('0') + assert Precise('5.534000') == Precise('5.5340') + + assert min(Precise('-3.1415'), Precise('-2')) == '-3.1415' + + assert max(Precise('3.1415'), Precise('-2')) == '3.1415' + + assert Precise('2') > Precise('1.2345') + assert not Precise('-3.1415') > Precise('-2') + assert not Precise('3.1415') > Precise('3.1415') + assert Precise.string_gt('3.14150000000000000000001', '3.1415') + + assert Precise('3.1415') >= Precise('3.1415') + assert Precise('3.14150000000000000000001') >= Precise('3.1415') + + assert not Precise('3.1415') < Precise('3.1415') + + assert Precise('3.1415') <= Precise('3.1415') + assert Precise('3.1415') <= Precise('3.14150000000000000000001') From 9607d0427903a05334576f7ce33e3d4e92282a51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Apr 2022 09:55:35 +0200 Subject: [PATCH 41/53] Improve ccxt imports --- freqtrade/exchange/exchange.py | 4 +--- setup.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 8bbbf6d4d..d2766cd6d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -16,9 +16,7 @@ import arrow import ccxt import ccxt.async_support as ccxt_async from cachetools import TTLCache -from ccxt import Precise -from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, - decimal_to_precision) +from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, Precise, decimal_to_precision from pandas import DataFrame from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell, diff --git a/setup.py b/setup.py index c5e418d0d..fadd4629f 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ setup( ], install_requires=[ # from requirements.txt - 'ccxt>=1.79.69', + 'ccxt>=1.80.67', 'SQLAlchemy', 'python-telegram-bot>=13.4', 'arrow>=0.17.0', From a1048fb619e186d83ff83ad25ba0e02e818d6fd1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 1 May 2022 17:00:00 +0200 Subject: [PATCH 42/53] Store monthly candles as "Mo" --- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/idatahandler.py | 16 ++++++++++++++-- freqtrade/data/history/jsondatahandler.py | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 23120a4ba..165685960 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -40,7 +40,7 @@ class HDF5DataHandler(IDataHandler): return [ ( cls.rebuild_pair_from_filename(match[1]), - match[2], + cls.rebuild_timeframe_from_filename(match[2]), CandleType.from_string(match[3]) ) for match in _tmp if match and len(match.groups()) > 1] diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 2e6b070ca..bd795f480 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class IDataHandler(ABC): - _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)' + _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)' def __init__(self, datadir: Path) -> None: self._datadir = datadir @@ -201,7 +201,7 @@ class IDataHandler(ABC): datadir = datadir.joinpath('futures') candle = f"-{candle_type}" filename = datadir.joinpath( - f'{pair_s}-{timeframe}{candle}.{cls._get_file_extension()}') + f'{pair_s}-{cls.timeframe_to_file(timeframe)}{candle}.{cls._get_file_extension()}') return filename @classmethod @@ -210,6 +210,18 @@ class IDataHandler(ABC): filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') return filename + @staticmethod + def timeframe_to_file(timeframe: str): + return timeframe.replace('M', 'Mo') + + @staticmethod + def rebuild_timeframe_from_filename(timeframe: str) -> str: + """ + converts timeframe from disk to file + Replaces mo with M (to avoid problems on case-insensitive filesystems) + """ + return re.sub('mo', 'M', timeframe, flags=re.IGNORECASE) + @staticmethod def rebuild_pair_from_filename(pair: str) -> str: """ diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 23054ac51..fa02c770b 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -41,7 +41,7 @@ class JsonDataHandler(IDataHandler): return [ ( cls.rebuild_pair_from_filename(match[1]), - match[2], + cls.rebuild_timeframe_from_filename(match[2]), CandleType.from_string(match[3]) ) for match in _tmp if match and len(match.groups()) > 1] From 2e65a1793d086ecbdf904d74a0dc3dc3b4ddeac1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 1 May 2022 19:51:25 +0200 Subject: [PATCH 43/53] Add fallback to load 1M files as well as 1Mo files --- freqtrade/data/history/hdf5datahandler.py | 11 +++++++--- freqtrade/data/history/idatahandler.py | 7 ++++--- freqtrade/data/history/jsondatahandler.py | 12 ++++++++--- tests/data/test_history.py | 25 ++++++++++++----------- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 165685960..6099c22bc 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -77,7 +77,8 @@ class HDF5DataHandler(IDataHandler): key = self._pair_ohlcv_key(pair, timeframe) _data = data.copy() - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) self.create_dir_if_needed(filename) _data.loc[:, self._columns].to_hdf( @@ -104,12 +105,16 @@ class HDF5DataHandler(IDataHandler): filename = self._pair_data_filename( self._datadir, pair, - timeframe, + self.timeframe_to_file(timeframe), candle_type=candle_type ) if not filename.exists(): - return pd.DataFrame(columns=self._columns) + # Fallback mode for 1M files + filename = self._pair_data_filename( + self._datadir, pair, timeframe, candle_type=candle_type) + if not filename.exists(): + return pd.DataFrame(columns=self._columns) where = [] if timerange: if timerange.starttype == 'date': diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index bd795f480..69d6212ee 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -98,7 +98,8 @@ class IDataHandler(ABC): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: True when deleted, false if file did not exist. """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) if filename.exists(): filename.unlink() return True @@ -201,7 +202,7 @@ class IDataHandler(ABC): datadir = datadir.joinpath('futures') candle = f"-{candle_type}" filename = datadir.joinpath( - f'{pair_s}-{cls.timeframe_to_file(timeframe)}{candle}.{cls._get_file_extension()}') + f'{pair_s}-{timeframe}{candle}.{cls._get_file_extension()}') return filename @classmethod @@ -220,7 +221,7 @@ class IDataHandler(ABC): converts timeframe from disk to file Replaces mo with M (to avoid problems on case-insensitive filesystems) """ - return re.sub('mo', 'M', timeframe, flags=re.IGNORECASE) + return re.sub('1mo', '1M', timeframe, flags=re.IGNORECASE) @staticmethod def rebuild_pair_from_filename(pair: str) -> str: diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index fa02c770b..38402a113 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -77,7 +77,8 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: None """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int @@ -103,9 +104,14 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type=candle_type) if not filename.exists(): - return DataFrame(columns=self._columns) + # Fallback mode for 1M files + filename = self._pair_data_filename( + self._datadir, pair, timeframe, candle_type=candle_type) + if not filename.exists(): + return DataFrame(columns=self._columns) try: pairdata = read_json(filename, orient='values') pairdata.columns = self._columns diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 82d4a841c..1e7d8855e 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -158,21 +158,22 @@ def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) -@pytest.mark.parametrize("pair,expected_result,candle_type", [ - ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json', ""), - ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json', ""), - ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json', ""), - (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""), - ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""), - ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""), - ("ETH/BTC", 'freqtrade/hello/world/futures/ETH_BTC-5m-mark.json', "mark"), - ("ACC_OLD/BTC", 'freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json', "index"), +@pytest.mark.parametrize("pair,timeframe,expected_result,candle_type", [ + ("ETH/BTC", "5m", "freqtrade/hello/world/ETH_BTC-5m.json", ""), + ("ETH/USDT", "1M", "freqtrade/hello/world/ETH_USDT-1Mo.json", ""), + ("Fabric Token/ETH", "5m", "freqtrade/hello/world/Fabric_Token_ETH-5m.json", ""), + ("ETHH20", "5m", "freqtrade/hello/world/ETHH20-5m.json", ""), + (".XBTBON2H", "5m", "freqtrade/hello/world/_XBTBON2H-5m.json", ""), + ("ETHUSD.d", "5m", "freqtrade/hello/world/ETHUSD_d-5m.json", ""), + ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/ACC_OLD_BTC-5m.json", ""), + ("ETH/BTC", "5m", "freqtrade/hello/world/futures/ETH_BTC-5m-mark.json", "mark"), + ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json", "index"), ]) -def test_json_pair_data_filename(pair, expected_result, candle_type): +def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - '5m', + JsonDataHandler.timeframe_to_file(timeframe), CandleType.from_string(candle_type) ) assert isinstance(fn, Path) @@ -180,7 +181,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type): fn = JsonGzDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - '5m', + JsonGzDataHandler.timeframe_to_file(timeframe), candle_type=CandleType.from_string(candle_type) ) assert isinstance(fn, Path) From 76637d3939994219e1ec15e1cdf7e513217536a5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 May 2022 19:53:01 +0200 Subject: [PATCH 44/53] Simplify timeframe-transition --- freqtrade/data/history/hdf5datahandler.py | 7 +++---- freqtrade/data/history/idatahandler.py | 9 ++++++--- freqtrade/data/history/jsondatahandler.py | 7 +++---- tests/data/test_history.py | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 6099c22bc..dadc9c7e6 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -77,8 +77,7 @@ class HDF5DataHandler(IDataHandler): key = self._pair_ohlcv_key(pair, timeframe) _data = data.copy() - filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) self.create_dir_if_needed(filename) _data.loc[:, self._columns].to_hdf( @@ -105,14 +104,14 @@ class HDF5DataHandler(IDataHandler): filename = self._pair_data_filename( self._datadir, pair, - self.timeframe_to_file(timeframe), + timeframe, candle_type=candle_type ) if not filename.exists(): # Fallback mode for 1M files filename = self._pair_data_filename( - self._datadir, pair, timeframe, candle_type=candle_type) + self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True) if not filename.exists(): return pd.DataFrame(columns=self._columns) where = [] diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 69d6212ee..07dc7c763 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -98,8 +98,7 @@ class IDataHandler(ABC): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: True when deleted, false if file did not exist. """ - filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) if filename.exists(): filename.unlink() return True @@ -194,10 +193,14 @@ class IDataHandler(ABC): datadir: Path, pair: str, timeframe: str, - candle_type: CandleType + candle_type: CandleType, + no_timeframe_modify: bool = False ) -> Path: pair_s = misc.pair_to_filename(pair) candle = "" + if not no_timeframe_modify: + timeframe = cls.timeframe_to_file(timeframe) + if candle_type != CandleType.SPOT: datadir = datadir.joinpath('futures') candle = f"-{candle_type}" diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 38402a113..83ec183df 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -77,8 +77,7 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: None """ - filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int @@ -105,11 +104,11 @@ class JsonDataHandler(IDataHandler): :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type=candle_type) + self._datadir, pair, timeframe, candle_type=candle_type) if not filename.exists(): # Fallback mode for 1M files filename = self._pair_data_filename( - self._datadir, pair, timeframe, candle_type=candle_type) + self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True) if not filename.exists(): return DataFrame(columns=self._columns) try: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 1e7d8855e..9709e7ad0 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -173,7 +173,7 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - JsonDataHandler.timeframe_to_file(timeframe), + timeframe, CandleType.from_string(candle_type) ) assert isinstance(fn, Path) @@ -181,7 +181,7 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonGzDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - JsonGzDataHandler.timeframe_to_file(timeframe), + timeframe, candle_type=CandleType.from_string(candle_type) ) assert isinstance(fn, Path) From fb7c0792c017f9deae8a397272ba310eca798664 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 17 May 2022 01:41:01 +0300 Subject: [PATCH 45/53] Track trade entries canceled by user. --- freqtrade/optimize/backtesting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 64107ae18..933cc2aea 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -297,6 +297,7 @@ class Backtesting: self.rejected_trades = 0 self.timedout_entry_orders = 0 self.timedout_exit_orders = 0 + self.canceled_trade_entries = 0 self.dataprovider.clear_cache() if enable_protections: self._load_protections(self.strategy) @@ -884,6 +885,7 @@ class Backtesting: return True elif self.check_order_replace(trade, order, current_time, row): # delete trade due to user request + self.canceled_trade_entries += 1 return True # default maintain trade return False @@ -1087,6 +1089,7 @@ class Backtesting: 'rejected_signals': self.rejected_trades, 'timedout_entry_orders': self.timedout_entry_orders, 'timedout_exit_orders': self.timedout_exit_orders, + 'canceled_trade_entries': self.canceled_trade_entries, 'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']), } From f2e2e57237a3335431f48e8fdb6557729631b926 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 17 May 2022 01:41:31 +0300 Subject: [PATCH 46/53] Report trade entries canceled by user. --- freqtrade/optimize/optimize_reports.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 42db366a1..7ee25ea73 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -468,6 +468,7 @@ def generate_strategy_stats(pairlist: List[str], 'rejected_signals': content['rejected_signals'], 'timedout_entry_orders': content['timedout_entry_orders'], 'timedout_exit_orders': content['timedout_exit_orders'], + 'canceled_trade_entries': content['canceled_trade_entries'], 'max_open_trades': max_open_trades, 'max_open_trades_setting': (config['max_open_trades'] if config['max_open_trades'] != float('inf') else -1), @@ -801,6 +802,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Entry/Exit Timeouts', f"{strat_results.get('timedout_entry_orders', 'N/A')} / " f"{strat_results.get('timedout_exit_orders', 'N/A')}"), + ('Canceled Trade Entries', strat_results.get('canceled_trade_entries', 'N/A')), ('', ''), # Empty line to improve readability ('Min balance', round_coin_value(strat_results['csum_min'], From 99aea454b5eaae99962718fbe1abf65cc63debb1 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 17 May 2022 01:42:48 +0300 Subject: [PATCH 47/53] Update testcases to match reporting. --- tests/optimize/test_backtesting.py | 8 ++++++++ tests/optimize/test_hyperopt.py | 2 ++ tests/optimize/test_optimize_reports.py | 2 ++ 3 files changed, 12 insertions(+) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index c87a0ef73..5b080fb11 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1168,6 +1168,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'final_balance': 1000, }) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', @@ -1280,6 +1281,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'final_balance': 1000, }, { @@ -1289,6 +1291,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'final_balance': 1000, } ]) @@ -1431,6 +1434,7 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker, 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'final_balance': 1000, }, { @@ -1440,6 +1444,7 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker, 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'final_balance': 1000, } ]) @@ -1534,6 +1539,7 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'final_balance': 1000, }, { @@ -1543,6 +1549,7 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'final_balance': 1000, } ]) @@ -1606,6 +1613,7 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'final_balance': 1000, }) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 75944390e..1d729190e 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -368,6 +368,7 @@ def test_hyperopt_format_results(hyperopt): 'rejected_signals': 2, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'backtest_start_time': 1619718665, 'backtest_end_time': 1619718665, } @@ -438,6 +439,7 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'final_balance': 1000, } diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index ff8d420b3..6ba03f08e 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -87,6 +87,7 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'backtest_start_time': Arrow.utcnow().int_timestamp, 'backtest_end_time': Arrow.utcnow().int_timestamp, 'run_id': '123', @@ -139,6 +140,7 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'rejected_signals': 20, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, 'backtest_start_time': Arrow.utcnow().int_timestamp, 'backtest_end_time': Arrow.utcnow().int_timestamp, 'run_id': '124', From a2a8e4fdc75f1a497471f1892fcd148b5052d1bc Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 17 May 2022 02:01:27 +0300 Subject: [PATCH 48/53] Update doc BT sample report. --- docs/backtesting.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index 02d1a53d1..6f0ed8447 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -320,6 +320,7 @@ A backtesting result will look like that: | Avg. Duration Loser | 6:55:00 | | Rejected Entry signals | 3089 | | Entry/Exit Timeouts | 0 / 0 | +| Canceled Trade Entries | 123 | | | | | Min balance | 0.00945123 BTC | | Max balance | 0.01846651 BTC | From 905b24bd4d4de6be7bdcf81011fd7fe37bd34678 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 17 May 2022 02:04:45 +0300 Subject: [PATCH 49/53] Update BT report detailing. --- docs/backtesting.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index 6f0ed8447..45c2704a0 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -417,6 +417,7 @@ It contains some useful key metrics about performance of your strategy on backte | Avg. Duration Loser | 6:55:00 | | Rejected Entry signals | 3089 | | Entry/Exit Timeouts | 0 / 0 | +| Canceled Trade Entries | 123 | | | | | Min balance | 0.00945123 BTC | | Max balance | 0.01846651 BTC | @@ -448,6 +449,7 @@ It contains some useful key metrics about performance of your strategy on backte - `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades. - `Rejected Entry signals`: Trade entry signals that could not be acted upon due to `max_open_trades` being reached. - `Entry/Exit Timeouts`: Entry/exit orders which did not fill (only applicable if custom pricing is used). +- `Canceled Trade Entries`: Number of trades that have been canceled by user request via `adjust_entry_price`. - `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period. - `Max % of account underwater`: Maximum percentage your account has decreased from the top since the simulation started. Calculated as the maximum of `(Max Balance - Current Balance) / (Max Balance)`. From 6e8f24f6a7cd634b9ce6d83ffc7871fee4efb9b8 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 17 May 2022 14:07:02 +0300 Subject: [PATCH 50/53] BT: track canceled/replaced orders also. --- freqtrade/optimize/backtesting.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 933cc2aea..9aee1215f 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -298,6 +298,8 @@ class Backtesting: self.timedout_entry_orders = 0 self.timedout_exit_orders = 0 self.canceled_trade_entries = 0 + self.canceled_entry_orders = 0 + self.replaced_entry_orders = 0 self.dataprovider.clear_cache() if enable_protections: self._load_protections(self.strategy) @@ -935,6 +937,7 @@ class Backtesting: return False else: del trade.orders[trade.orders.index(order)] + self.canceled_entry_orders += 1 # place new order if result was not None if requested_rate: @@ -942,6 +945,7 @@ class Backtesting: requested_rate=requested_rate, requested_stake=(order.remaining * order.price), direction='short' if trade.is_short else 'long') + self.replaced_entry_orders += 1 else: # assumption: there can't be multiple open entry orders at any given time return (trade.nr_of_successful_entries == 0) @@ -1090,6 +1094,8 @@ class Backtesting: 'timedout_entry_orders': self.timedout_entry_orders, 'timedout_exit_orders': self.timedout_exit_orders, 'canceled_trade_entries': self.canceled_trade_entries, + 'canceled_entry_orders': self.canceled_entry_orders, + 'replaced_entry_orders': self.replaced_entry_orders, 'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']), } From 0585b378b3187926ca6e0980c8c67eed7dd1378d Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 17 May 2022 14:07:42 +0300 Subject: [PATCH 51/53] BT: Report canceled/replaced orders also. --- freqtrade/optimize/optimize_reports.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 7ee25ea73..93336fa3f 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -469,6 +469,8 @@ def generate_strategy_stats(pairlist: List[str], 'timedout_entry_orders': content['timedout_entry_orders'], 'timedout_exit_orders': content['timedout_exit_orders'], 'canceled_trade_entries': content['canceled_trade_entries'], + 'canceled_entry_orders': content['canceled_entry_orders'], + 'replaced_entry_orders': content['replaced_entry_orders'], 'max_open_trades': max_open_trades, 'max_open_trades_setting': (config['max_open_trades'] if config['max_open_trades'] != float('inf') else -1), @@ -754,6 +756,12 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Drawdown End', strat_results['drawdown_end']), ]) + entry_adjustment_metrics = [ + ('Canceled Trade Entries', strat_results.get('canceled_trade_entries', 'N/A')), + ('Canceled Entry Orders', strat_results.get('canceled_entry_orders', 'N/A')), + ('Replaced Entry Orders', strat_results.get('replaced_entry_orders', 'N/A')), + ] if strat_results.get('canceled_entry_orders', 0) > 0 else [] + # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show # command stores these results and newer version of freqtrade must be able to handle old # results with missing new fields. @@ -802,7 +810,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Entry/Exit Timeouts', f"{strat_results.get('timedout_entry_orders', 'N/A')} / " f"{strat_results.get('timedout_exit_orders', 'N/A')}"), - ('Canceled Trade Entries', strat_results.get('canceled_trade_entries', 'N/A')), + *entry_adjustment_metrics, ('', ''), # Empty line to improve readability ('Min balance', round_coin_value(strat_results['csum_min'], From bb7ffd8fbec123de522f43909603c3c80b4c899c Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 17 May 2022 14:08:35 +0300 Subject: [PATCH 52/53] Update testcases relying on BT results. --- tests/optimize/test_backtesting.py | 16 ++++++++++++++++ tests/optimize/test_hyperopt.py | 4 ++++ tests/optimize/test_optimize_reports.py | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 5b080fb11..f169e0a35 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1169,6 +1169,8 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'final_balance': 1000, }) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', @@ -1282,6 +1284,8 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'final_balance': 1000, }, { @@ -1292,6 +1296,8 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'final_balance': 1000, } ]) @@ -1435,6 +1441,8 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'final_balance': 1000, }, { @@ -1445,6 +1453,8 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'final_balance': 1000, } ]) @@ -1540,6 +1550,8 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'final_balance': 1000, }, { @@ -1550,6 +1562,8 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'final_balance': 1000, } ]) @@ -1614,6 +1628,8 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'final_balance': 1000, }) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 1d729190e..dcc1ddeea 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -369,6 +369,8 @@ def test_hyperopt_format_results(hyperopt): 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'backtest_start_time': 1619718665, 'backtest_end_time': 1619718665, } @@ -440,6 +442,8 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'final_balance': 1000, } diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 6ba03f08e..997c0436e 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -88,6 +88,8 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'backtest_start_time': Arrow.utcnow().int_timestamp, 'backtest_end_time': Arrow.utcnow().int_timestamp, 'run_id': '123', @@ -141,6 +143,8 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'timedout_entry_orders': 0, 'timedout_exit_orders': 0, 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, 'backtest_start_time': Arrow.utcnow().int_timestamp, 'backtest_end_time': Arrow.utcnow().int_timestamp, 'run_id': '124', From c6bf6779f874c1d5c3fe3c8065bfe01516c5a2ae Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 17 May 2022 14:09:01 +0300 Subject: [PATCH 53/53] Update docs BT sample report and details. --- docs/backtesting.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 45c2704a0..b4d9aef80 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -320,7 +320,9 @@ A backtesting result will look like that: | Avg. Duration Loser | 6:55:00 | | Rejected Entry signals | 3089 | | Entry/Exit Timeouts | 0 / 0 | -| Canceled Trade Entries | 123 | +| Canceled Trade Entries | 34 | +| Canceled Entry Orders | 123 | +| Replaced Entry Orders | 89 | | | | | Min balance | 0.00945123 BTC | | Max balance | 0.01846651 BTC | @@ -417,7 +419,9 @@ It contains some useful key metrics about performance of your strategy on backte | Avg. Duration Loser | 6:55:00 | | Rejected Entry signals | 3089 | | Entry/Exit Timeouts | 0 / 0 | -| Canceled Trade Entries | 123 | +| Canceled Trade Entries | 34 | +| Canceled Entry Orders | 123 | +| Replaced Entry Orders | 89 | | | | | Min balance | 0.00945123 BTC | | Max balance | 0.01846651 BTC | @@ -450,6 +454,8 @@ It contains some useful key metrics about performance of your strategy on backte - `Rejected Entry signals`: Trade entry signals that could not be acted upon due to `max_open_trades` being reached. - `Entry/Exit Timeouts`: Entry/exit orders which did not fill (only applicable if custom pricing is used). - `Canceled Trade Entries`: Number of trades that have been canceled by user request via `adjust_entry_price`. +- `Canceled Entry Orders`: Number of entry orders that have been canceled by user request via `adjust_entry_price`. +- `Replaced Entry Orders`: Number of entry orders that have been replaced by user request via `adjust_entry_price`. - `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period. - `Max % of account underwater`: Maximum percentage your account has decreased from the top since the simulation started. Calculated as the maximum of `(Max Balance - Current Balance) / (Max Balance)`.