From 3c8387ab611df10c8a327b5196f4798541f67b10 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Fri, 1 Apr 2022 20:48:13 +0900 Subject: [PATCH 01/35] Add exchange id for binance Futures --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index b808096d2..420061050 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -87,7 +87,7 @@ When trading on Binance Futures market, orderbook must be used because there is Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. -* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`. +* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance` for spot market, and use `binanceusdm` or `binancecoinm` for Futures market. * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`. ## Kraken From da0688b6aa26f6eee36acef9ede4186b48d876c6 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 2 Apr 2022 03:20:21 +0900 Subject: [PATCH 02/35] Revert "Add exchange id for binance Futures" This reverts commit 3c8387ab611df10c8a327b5196f4798541f67b10. --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 420061050..b808096d2 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -87,7 +87,7 @@ When trading on Binance Futures market, orderbook must be used because there is Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. -* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance` for spot market, and use `binanceusdm` or `binancecoinm` for Futures market. +* [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`. * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`. ## Kraken From 8a3c2c6cad8beef24900f1939e2fbdc7736c365d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 13 May 2022 19:32:52 +0530 Subject: [PATCH 03/35] Corrected docstring Discussed in Discord --- freqtrade/persistence/trade_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index bb8c03dd2..b1fff5cf3 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -656,7 +656,7 @@ class LocalTrade(): def recalc_open_trade_value(self) -> None: """ Recalculate open_trade_value. - Must be called whenever open_rate, fee_open or is_short is changed. + Must be called whenever open_rate, fee_open is changed. """ self.open_trade_value = self._calc_open_trade_value() From 9d13c8729257ddfb9e8a64daf6bd8b99a9010fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 13 May 2022 21:46:25 +0530 Subject: [PATCH 04/35] cleaned up backtesting Solves the [bug](https://github.com/freqtrade/freqtrade/runs/6425715015?check_suite_focus=true) --- tests/optimize/test_backtest_detail.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 18b4c3621..0441d4214 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -522,7 +522,7 @@ tc32 = BTContainer(data=[ trailing_stop_positive=0.03, trades=[ BTrade(exit_reason=ExitType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3, is_short=True) - ] +] ) # Test 33: trailing_stop should be triggered by low of next candle, without adjusting stoploss using @@ -662,7 +662,7 @@ tc41 = BTContainer(data=[ custom_entry_price=4000, trades=[ BTrade(exit_reason=ExitType.STOP_LOSS, open_tick=1, close_tick=1, is_short=True) - ] +] ) # Test 42: Custom-entry-price around candle low @@ -933,3 +933,5 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) assert res.open_date == _get_frame_time_from_offset(trade.open_tick) assert res.close_date == _get_frame_time_from_offset(trade.close_tick) assert res.is_short == trade.is_short + backtesting.cleanup() + del backtesting From 64668b11dab7530f7606e9a9deee458689faaefe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 09:10:38 +0200 Subject: [PATCH 05/35] add ohlcv_has_history - disabling kraken downloads --- freqtrade/commands/data_commands.py | 6 ++++++ freqtrade/exchange/exchange.py | 13 +++++++++---- freqtrade/exchange/kraken.py | 1 + tests/commands/test_commands.py | 17 +++++++++++++++++ tests/exchange/test_ccxt_compat.py | 2 +- tests/exchange/test_exchange.py | 7 +++++++ 6 files changed, 41 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index a2e2a100a..61a99782e 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -79,6 +79,12 @@ def start_download_data(args: Dict[str, Any]) -> None: data_format_trades=config['dataformat_trades'], ) else: + if not exchange._ft_has.get('ohlcv_has_history', True): + raise OperationalException( + f"Historic klines not available for {exchange.name}. " + "Please use `--dl-trades` instead for this exchange " + "(will unfortunately take a long time)." + ) pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 65b9fb628..8d74a8446 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -64,6 +64,7 @@ class Exchange: "time_in_force_parameter": "timeInForce", "ohlcv_params": {}, "ohlcv_candle_limit": 500, + "ohlcv_has_history": True, # Some exchanges (Kraken) don't provide history via ohlcv "ohlcv_partial_candle": True, "ohlcv_require_since": False, # Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency @@ -621,13 +622,17 @@ class Exchange: # Allow 5 calls to the exchange per pair required_candle_call_count = int( (candle_count / candle_limit) + (0 if candle_count % candle_limit == 0 else 1)) + if self._ft_has['ohlcv_has_history']: - if required_candle_call_count > 5: - # Only allow 5 calls per pair to somewhat limit the impact + if required_candle_call_count > 5: + # Only allow 5 calls per pair to somewhat limit the impact + raise OperationalException( + f"This strategy requires {startup_candles} candles to start, which is more than 5x " + f"the amount of candles {self.name} provides for {timeframe}.") + elif required_candle_call_count > 1: raise OperationalException( - f"This strategy requires {startup_candles} candles to start, which is more than 5x " + f"This strategy requires {startup_candles} candles to start, which is more than " f"the amount of candles {self.name} provides for {timeframe}.") - if required_candle_call_count > 1: logger.warning(f"Using {required_candle_call_count} calls to get OHLCV. " f"This can result in slower operations for the bot. Please check " diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 33a2c7f87..900f6c898 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -23,6 +23,7 @@ class Kraken(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, "ohlcv_candle_limit": 720, + "ohlcv_has_history": False, "trades_pagination": "id", "trades_pagination_arg": "since", "mark_ohlcv_timeframe": "4h", diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 0932f4362..b37edf9c7 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -835,6 +835,23 @@ def test_download_data_trades(mocker, caplog): start_download_data(pargs) +def test_download_data_data_invalid(mocker): + patch_exchange(mocker, id="kraken") + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + "--exchange", "kraken", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, match=r"Historic klines not available for .*"): + start_download_data(pargs) + + def test_start_convert_trades(mocker, caplog): convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index d8832bb71..ac7c8a528 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -219,7 +219,7 @@ class TestCCXTExchange(): assert len(l2['asks']) == next_limit assert len(l2['asks']) == next_limit - def test_fetch_ohlcv(self, exchange): + def test_ccxt_fetch_ohlcv(self, exchange): exchange, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] timeframe = EXCHANGES[exchangename]['timeframe'] diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 77a04ac6c..ed2a7b7ee 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1084,6 +1084,13 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog): with pytest.raises(OperationalException, match=r'This strategy requires 6000.*'): Exchange(default_conf) + # Emulate kraken mode + ex._ft_has['ohlcv_has_history'] = False + with pytest.raises(OperationalException, + match=r'This strategy requires 2500.*, ' + r'which is more than the amount.*'): + ex.validate_required_startup_candles(2500, '5m') + def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) From 111b04c9e65668067646265e614326f81aa1bf1c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 09:51:44 +0200 Subject: [PATCH 06/35] Okx - conditional candle-length --- freqtrade/exchange/exchange.py | 17 ++++++++++----- freqtrade/exchange/okx.py | 28 +++++++++++++++++++++++-- tests/exchange/test_ccxt_compat.py | 33 ++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 8d74a8446..864aa36e9 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -309,12 +309,15 @@ class Exchange: if self.log_responses: logger.info(f"API {endpoint}: {response}") - def ohlcv_candle_limit(self, timeframe: str) -> int: + def ohlcv_candle_limit( + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> int: """ Exchange ohlcv candle limit Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit :param timeframe: Timeframe to check + :param candle_type: Candle-type + :param since_ms: Candle-type :return: Candle limit as integer """ return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( @@ -616,7 +619,7 @@ class Exchange: Checks if required startup_candles is more than ohlcv_candle_limit(). Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. """ - candle_limit = self.ohlcv_candle_limit(timeframe) + candle_limit = self.ohlcv_candle_limit(timeframe, self._config['candle_type_def'], None) # Require one more candle - to account for the still open candle. candle_count = startup_candles + 1 # Allow 5 calls to the exchange per pair @@ -1708,7 +1711,8 @@ class Exchange: :param candle_type: Any of the enum CandleType (must match trading mode!) """ - one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) + one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit( + timeframe, candle_type, since_ms) logger.debug( "one_call: %s msecs (%s)", one_call, @@ -1744,7 +1748,8 @@ class Exchange: if (not since_ms and (self._ft_has["ohlcv_require_since"] or self.required_candle_call_count > 1)): # Multiple calls for one pair - to get more history - one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) + one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit( + timeframe, candle_type, since_ms) move_to = one_call * self.required_candle_call_count now = timeframe_to_next_date(timeframe) since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000) @@ -1862,7 +1867,9 @@ class Exchange: pair, timeframe, since_ms, s ) params = deepcopy(self._ft_has.get('ohlcv_params', {})) - candle_limit = self.ohlcv_candle_limit(timeframe) + candle_limit = self.ohlcv_candle_limit( + timeframe, candle_type=candle_type, since_ms=since_ms) + if candle_type != CandleType.SPOT: params.update({'price': candle_type}) if candle_type != CandleType.FUNDING_RATE: diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 654021182..5e24997d7 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -1,13 +1,16 @@ import logging -from typing import Dict, List, Tuple +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional, Tuple import ccxt from freqtrade.constants import BuySell from freqtrade.enums import MarginMode, TradingMode +from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier +from freqtrade.exchange.exchange import timeframe_to_minutes logger = logging.getLogger(__name__) @@ -20,7 +23,7 @@ class Okx(Exchange): """ _ft_has: Dict = { - "ohlcv_candle_limit": 100, + "ohlcv_candle_limit": 300, # Warning, special case with data prior to X months "mark_ohlcv_timeframe": "4h", "funding_fee_timeframe": "8h", } @@ -37,6 +40,27 @@ class Okx(Exchange): net_only = True + def ohlcv_candle_limit( + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> int: + """ + Exchange ohlcv candle limit + Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits + per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit + :param timeframe: Timeframe to check + :param candle_type: Candle-type + :param since_ms: Candle-type + :return: Candle limit as integer + """ + now = datetime.now(timezone.utc) + offset_mins = timeframe_to_minutes(timeframe) * self._ft_has['ohlcv_candle_limit'] + if since_ms and since_ms < ((now - timedelta(minutes=offset_mins)).timestamp() * 1000): + return 100 + if candle_type not in (CandleType.FUTURES, CandleType.SPOT): + return 100 + + return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( + timeframe, self._ft_has.get('ohlcv_candle_limit'))) + @retrier def additional_exchange_init(self) -> None: """ diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index ac7c8a528..ea9a166f6 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -13,6 +13,7 @@ import pytest from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date +from freqtrade.exchange.exchange import timeframe_to_msecs from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_default_conf_usdt @@ -236,6 +237,38 @@ class TestCCXTExchange(): now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2)) assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) + def test_ccxt__async_get_candle_history(self, exchange): + exchange, exchangename = exchange + # For some weired reason, this test returns random lengths for bittrex. + if not exchange._ft_has['ohlcv_has_history'] or exchangename == 'bittrex': + return + pair = EXCHANGES[exchangename]['pair'] + timeframe = EXCHANGES[exchangename]['timeframe'] + candle_type = CandleType.SPOT + timeframe_ms = timeframe_to_msecs(timeframe) + now = timeframe_to_prev_date( + timeframe, datetime.now(timezone.utc)) + for offset in (360, 120, 30, 10, 5, 2): + since = now - timedelta(days=offset) + since_ms = int(since.timestamp() * 1000) + + res = exchange.loop.run_until_complete(exchange._async_get_candle_history( + pair=pair, + timeframe=timeframe, + since_ms=since_ms, + candle_type=candle_type + ) + ) + assert res + assert res[0] == pair + assert res[1] == timeframe + assert res[2] == candle_type + candles = res[3] + candle_count = exchange.ohlcv_candle_limit(timeframe, candle_type, since_ms) * 0.9 + candle_count1 = (now.timestamp() * 1000 - since_ms) // timeframe_ms + assert len(candles) >= min(candle_count, candle_count1) + assert candles[0][0] == since_ms or (since_ms + timeframe_ms) + def test_ccxt_fetch_funding_rate_history(self, exchange_futures): exchange, exchangename = exchange_futures if not exchange: From bb1b283d9548ea58cee0d588d48e631a015f8297 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 13:27:36 +0200 Subject: [PATCH 07/35] Update some ohlcv_candle_limit calls --- freqtrade/exchange/exchange.py | 2 +- freqtrade/exchange/okx.py | 2 +- freqtrade/plugins/pairlist/AgeFilter.py | 9 +++++---- freqtrade/plugins/pairlist/VolatilityFilter.py | 6 +++--- freqtrade/plugins/pairlist/VolumePairList.py | 7 ++++--- freqtrade/plugins/pairlist/rangestabilityfilter.py | 6 +++--- tests/exchange/test_ccxt_compat.py | 3 ++- tests/exchange/test_exchange.py | 8 ++++---- 8 files changed, 23 insertions(+), 20 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 864aa36e9..d2a01f394 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -310,7 +310,7 @@ class Exchange: logger.info(f"API {endpoint}: {response}") def ohlcv_candle_limit( - self, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> int: + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: """ Exchange ohlcv candle limit Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 5e24997d7..6d25bb12b 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -41,7 +41,7 @@ class Okx(Exchange): net_only = True def ohlcv_candle_limit( - self, timeframe: str, candle_type: CandleType, since_ms: Optional[int]) -> int: + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: """ Exchange ohlcv candle limit Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index bb6f75012..418c0f14e 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -32,18 +32,19 @@ class AgeFilter(IPairList): self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._max_days_listed = pairlistconfig.get('max_days_listed', None) + candle_limit = exchange.ohlcv_candle_limit('1d', self._config['candle_type_def']) if self._min_days_listed < 1: raise OperationalException("AgeFilter requires min_days_listed to be >= 1") - if self._min_days_listed > exchange.ohlcv_candle_limit('1d'): + if self._min_days_listed > candle_limit: raise OperationalException("AgeFilter requires min_days_listed to not exceed " "exchange max request size " - f"({exchange.ohlcv_candle_limit('1d')})") + f"({candle_limit})") if self._max_days_listed and self._max_days_listed <= self._min_days_listed: raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted") - if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'): + if self._max_days_listed and self._max_days_listed > candle_limit: raise OperationalException("AgeFilter requires max_days_listed to not exceed " "exchange max request size " - f"({exchange.ohlcv_candle_limit('1d')})") + f"({candle_limit})") @property def needstickers(self) -> bool: diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 6aa857c2c..bab44bdd1 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -38,12 +38,12 @@ class VolatilityFilter(IPairList): self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) + candle_limit = exchange.ohlcv_candle_limit('1d', self._config['candle_type_def']) if self._days < 1: raise OperationalException("VolatilityFilter requires lookback_days to be >= 1") - if self._days > exchange.ohlcv_candle_limit('1d'): + if self._days > candle_limit: raise OperationalException("VolatilityFilter requires lookback_days to not " - "exceed exchange max request size " - f"({exchange.ohlcv_candle_limit('1d')})") + f"exceed exchange max request size ({candle_limit})") @property def needstickers(self) -> bool: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 26e7d45be..cd16a46a3 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -84,12 +84,13 @@ class VolumePairList(IPairList): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') + candle_limit = exchange.ohlcv_candle_limit( + self._lookback_timeframe, self._config['candle_type_def']) if self._lookback_period < 0: raise OperationalException("VolumeFilter requires lookback_period to be >= 0") - if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe): + if self._lookback_period > candle_limit: raise OperationalException("VolumeFilter requires lookback_period to not " - "exceed exchange max request size " - f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})") + f"exceed exchange max request size ({candle_limit})") @property def needstickers(self) -> bool: diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index c9edfd13d..de016c3a6 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -33,12 +33,12 @@ class RangeStabilityFilter(IPairList): self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) + candle_limit = exchange.ohlcv_candle_limit('1d', self._config['candle_type_def']) if self._days < 1: raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1") - if self._days > exchange.ohlcv_candle_limit('1d'): + if self._days > candle_limit: raise OperationalException("RangeStabilityFilter requires lookback_days to not " - "exceed exchange max request size " - f"({exchange.ohlcv_candle_limit('1d')})") + f"exceed exchange max request size ({candle_limit})") @property def needstickers(self) -> bool: diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index ea9a166f6..e016873cb 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -232,7 +232,8 @@ class TestCCXTExchange(): assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf)) # assert len(exchange.klines(pair_tf)) > 200 # Assume 90% uptime ... - assert len(exchange.klines(pair_tf)) > exchange.ohlcv_candle_limit(timeframe) * 0.90 + assert len(exchange.klines(pair_tf)) > exchange.ohlcv_candle_limit( + timeframe, CandleType.SPOT) * 0.90 # Check if last-timeframe is within the last 2 intervals now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2)) assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ed2a7b7ee..9d7b77a8e 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1882,7 +1882,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls - since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 + since = 5 * 60 * exchange.ohlcv_candle_limit('5m', CandleType.SPOT) * 1.8 ret = exchange.get_historic_ohlcv( pair, "5m", @@ -1948,7 +1948,7 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name, candle_ty exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls - since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 + since = 5 * 60 * exchange.ohlcv_candle_limit('5m', CandleType.SPOT) * 1.8 ret = exchange.get_historic_ohlcv_as_df( pair, "5m", @@ -2002,7 +2002,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ ) # Required candles candles = (end_ts - start_ts) / 300_000 - exp = candles // exchange.ohlcv_candle_limit('5m') + 1 + exp = candles // exchange.ohlcv_candle_limit('5m', CandleType.SPOT) + 1 # Depending on the exchange, this should be called between 1 and 6 times. assert exchange._api_async.fetch_ohlcv.call_count == exp @@ -3349,7 +3349,7 @@ def test_ohlcv_candle_limit(default_conf, mocker, exchange_name): expected = exchange._ft_has['ohlcv_candle_limit_per_timeframe'][timeframe] # This should only run for bittrex assert exchange_name == 'bittrex' - assert exchange.ohlcv_candle_limit(timeframe) == expected + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT) == expected def test_timeframe_to_minutes(): From 2a1368d508a621a35579bed55db1d6eb841933ee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 14:14:53 +0200 Subject: [PATCH 08/35] Offsetfilter: add number_assets parameter closes #6824 --- docs/includes/pairlists.md | 8 ++++---- freqtrade/plugins/pairlist/OffsetFilter.py | 9 ++++++++- tests/plugins/test_pairlist.py | 10 +++++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 6acd361fe..0f55c1b79 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -160,17 +160,17 @@ This filter allows freqtrade to ignore pairs until they have been listed for at Offsets an incoming pairlist by a given `offset` value. -As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split -a larger pairlist on two bot instances. +As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split a larger pairlist on two bot instances. -Example to remove the first 10 pairs from the pairlist: +Example to remove the first 10 pairs from the pairlist, and takes the next 20 (taking items 10-30 of the initial list): ```json "pairlists": [ // ... { "method": "OffsetFilter", - "offset": 10 + "offset": 10, + "number_assets": 20 } ], ``` diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index 573a573a6..e0f8414ef 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -19,6 +19,7 @@ class OffsetFilter(IPairList): super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) self._offset = pairlistconfig.get('offset', 0) + self._number_pairs = pairlistconfig.get('number_assets', 0) if self._offset < 0: raise OperationalException("OffsetFilter requires offset to be >= 0") @@ -36,7 +37,9 @@ class OffsetFilter(IPairList): """ Short whitelist method description - used for startup-messages """ - return f"{self.name} - Offseting pairs by {self._offset}." + if self._number_pairs: + return f"{self.name} - Taking {self._number_pairs} Pairs, starting from {self._offset}." + return f"{self.name} - Offsetting pairs by {self._offset}." def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ @@ -50,5 +53,9 @@ class OffsetFilter(IPairList): self.log_once(f"Offset of {self._offset} is larger than " + f"pair count of {len(pairlist)}", logger.warning) pairs = pairlist[self._offset:] + if self._number_pairs: + pairs = pairs[:self._number_pairs] + self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info) + return pairs diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d80f23c8a..c29e619b1 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -470,12 +470,16 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "BTC", ['ETH/BTC', 'TKN/BTC']), # VolumePairList with no offset = unchanged pairlist ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, - {"method": "OffsetFilter", "offset": 0}], + {"method": "OffsetFilter", "offset": 0, "number_assets": 0}], "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']), # VolumePairList with offset = 2 ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, {"method": "OffsetFilter", "offset": 2}], "USDT", ['ADAHALF/USDT', 'ADADOUBLE/USDT']), + # VolumePairList with offset and limit + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 1, "number_assets": 2}], + "USDT", ['NANO/USDT', 'ADAHALF/USDT']), # VolumePairList with higher offset, than total pairlist ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, {"method": "OffsetFilter", "offset": 100}], @@ -1152,6 +1156,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo "0.01 and above 0.99 over the last days.'}]", None ), + ({"method": "OffsetFilter", "offset": 5, "number_assets": 10}, + "[{'OffsetFilter': 'OffsetFilter - Taking 10 Pairs, starting from 5.'}]", + None + ), ]) def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, desc_expected, exception_expected): From 5767d652bf9ffb97b4498c95cff9fc0e5c1654cf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 13:52:58 +0200 Subject: [PATCH 09/35] Add explicit test and document behavior --- freqtrade/exchange/exchange.py | 3 ++- freqtrade/exchange/okx.py | 8 +++++--- tests/exchange/test_okx.py | 27 +++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d2a01f394..86f80871b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -630,7 +630,8 @@ class Exchange: if required_candle_call_count > 5: # Only allow 5 calls per pair to somewhat limit the impact raise OperationalException( - f"This strategy requires {startup_candles} candles to start, which is more than 5x " + f"This strategy requires {startup_candles} candles to start, " + "which is more than 5x " f"the amount of candles {self.name} provides for {timeframe}.") elif required_candle_call_count > 1: raise OperationalException( diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 6d25bb12b..c8324e62e 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -41,11 +41,13 @@ class Okx(Exchange): net_only = True def ohlcv_candle_limit( - self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: + self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int: """ Exchange ohlcv candle limit - Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits - per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit + OKX has the following behaviour: + * 300 candles for uptodate data + * 100 candles for historic data + * 100 candles for additional candles (not futures or spot). :param timeframe: Timeframe to check :param candle_type: Candle-type :param since_ms: Candle-type diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index f6bdd35ad..2804d471a 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -1,12 +1,39 @@ +from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, PropertyMock import pytest from freqtrade.enums import MarginMode, TradingMode +from freqtrade.enums.candletype import CandleType +from freqtrade.exchange.exchange import timeframe_to_minutes from tests.conftest import get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers +def test_okx_ohlcv_candle_limit(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf, id='okx') + timeframes = ('1m', '5m', '1h') + start_time = int(datetime(2021, 1, 1, tzinfo=timezone.utc).timestamp() * 1000) + + for timeframe in timeframes: + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT) == 300 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES) == 300 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.MARK) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUNDING_RATE) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, start_time) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, start_time) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.MARK, start_time) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUNDING_RATE, start_time) == 100 + one_call = int((datetime.now(timezone.utc) - timedelta( + minutes=290 * timeframe_to_minutes(timeframe))).timestamp() * 1000) + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, one_call) == 300 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, one_call) == 300 + one_call = int((datetime.now(timezone.utc) - timedelta( + minutes=320 * timeframe_to_minutes(timeframe))).timestamp() * 1000) + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, one_call) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, one_call) == 100 + + def test_get_maintenance_ratio_and_amt_okx( default_conf, mocker, From 1c20fb7638d433e0ba703ff995c572a42ecb65ac Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 14 May 2022 16:37:04 +0300 Subject: [PATCH 10/35] Refresh open_rate and stoploss on order replacement. --- freqtrade/optimize/backtesting.py | 4 ++++ freqtrade/persistence/trade_model.py | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 45300b744..f439e4e63 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -780,6 +780,8 @@ class Backtesting: # interest_rate=interest_rate, orders=[], ) + elif trade.nr_of_successful_entries == 0: + trade.open_rate = propose_rate trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) @@ -940,6 +942,8 @@ class Backtesting: requested_rate=requested_rate, requested_stake=(order.remaining * order.price), direction='short' if trade.is_short else 'long') + trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, + initial=False, refresh=True) else: # assumption: there can't be multiple open entry orders at any given time return (trade.nr_of_successful_entries == 0) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index b1fff5cf3..28cca5532 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -491,7 +491,7 @@ class LocalTrade(): self.stoploss_last_update = datetime.utcnow() def adjust_stop_loss(self, current_price: float, stoploss: float, - initial: bool = False) -> None: + initial: bool = False, refresh: bool = False) -> None: """ This adjusts the stop loss to it's most recently observed setting :param current_price: Current rate the asset is traded @@ -516,8 +516,7 @@ class LocalTrade(): new_loss = max(self.liquidation_price, new_loss) # no stop loss assigned yet - if self.initial_stop_loss_pct is None: - logger.debug(f"{self.pair} - Assigning new stoploss...") + if self.initial_stop_loss_pct is None or refresh: self._set_stop_loss(new_loss, stoploss) self.initial_stop_loss = new_loss self.initial_stop_loss_pct = -1 * abs(stoploss) From ec54b47b6e36394f5f633af0f7d5a5cdc960ccfb Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 14 May 2022 16:39:27 +0300 Subject: [PATCH 11/35] Flake fix. --- tests/optimize/test_backtest_detail.py | 35 ++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 0441d4214..4b4c446e0 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -762,7 +762,7 @@ tc48 = BTContainer(data=[ [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust [3, 5100, 5100, 4650, 4750, 6172, 0, 1], [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], - stop_loss=-0.01, roi={"0": 0.10}, profit_perc=-0.087, + stop_loss=-0.2, roi={"0": 0.10}, profit_perc=-0.087, use_exit_signal=True, timeout=1000, custom_entry_price=4200, adjust_entry_price=5200, trades=[BTrade(exit_reason=ExitType.EXIT_SIGNAL, open_tick=1, close_tick=4, is_short=False)] @@ -777,7 +777,7 @@ tc49 = BTContainer(data=[ [2, 4900, 5250, 4900, 5100, 6172, 0, 0, 0, 0], # Order readjust [3, 5100, 5100, 4650, 4750, 6172, 0, 0, 0, 1], [4, 4750, 4950, 4350, 4750, 6172, 0, 0, 0, 0]], - stop_loss=-0.01, roi={"0": 0.10}, profit_perc=0.05, + stop_loss=-0.2, roi={"0": 0.10}, profit_perc=0.05, use_exit_signal=True, timeout=1000, custom_entry_price=5300, adjust_entry_price=5000, trades=[BTrade(exit_reason=ExitType.EXIT_SIGNAL, open_tick=1, close_tick=4, is_short=True)] @@ -811,6 +811,35 @@ tc51 = BTContainer(data=[ trades=[] ) +# Test 52: Custom-entry-price below all candles - readjust order - stoploss +tc52 = BTContainer(data=[ + # D O H L C V EL XL ES Xs BT + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], # stoploss hit? + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.03, roi={"0": 0.10}, profit_perc=-0.03, + use_exit_signal=True, timeout=1000, + custom_entry_price=4200, adjust_entry_price=5200, + trades=[BTrade(exit_reason=ExitType.STOP_LOSS, open_tick=1, close_tick=2, is_short=False)] +) + + +# Test 53: Custom-entry-price short above all candles - readjust order - stoploss +tc53 = BTContainer(data=[ + # D O H L C V EL XL ES Xs BT + [0, 5000, 5050, 4950, 5000, 6172, 0, 0, 1, 0], + [1, 5000, 5200, 4951, 5000, 6172, 0, 0, 0, 0], # enter trade (signal on last candle) + [2, 4900, 5250, 4900, 5100, 6172, 0, 0, 0, 0], # Order readjust + [3, 5100, 5100, 4650, 4750, 6172, 0, 0, 0, 1], # stoploss hit? + [4, 4750, 4950, 4350, 4750, 6172, 0, 0, 0, 0]], + stop_loss=-0.03, roi={"0": 0.10}, profit_perc=-0.03, + use_exit_signal=True, timeout=1000, + custom_entry_price=5300, adjust_entry_price=5000, + trades=[BTrade(exit_reason=ExitType.STOP_LOSS, open_tick=1, close_tick=2, is_short=True)] +) + TESTS = [ tc0, tc1, @@ -864,6 +893,8 @@ TESTS = [ tc49, tc50, tc51, + tc52, + tc53, ] From c27e0a0a1b8837525fe93cfaa459a78c4dce8f2b Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 14 May 2022 16:56:56 +0300 Subject: [PATCH 12/35] Allow SL refresh only if no filled entry orders. --- freqtrade/persistence/trade_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 28cca5532..bbdeeef47 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -502,6 +502,7 @@ class LocalTrade(): if initial and not (self.stop_loss is None or self.stop_loss == 0): # Don't modify if called with initial and nothing to do return + refresh = False if self.nr_of_successful_entries > 0 else refresh leverage = self.leverage or 1.0 if self.is_short: From 3b144392407501a7df9ecb32c04db6c434dbbcb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 16:16:32 +0200 Subject: [PATCH 13/35] Slightly improve performance of order adjusts Avoind 2nd call to `get_rate()`. closes #6821 --- freqtrade/freqtradebot.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 03cd322a6..07b055309 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -536,7 +536,8 @@ class FreqtradeBot(LoggingMixin): if stake_amount is not None and stake_amount > 0.0: # We should increase our position - self.execute_entry(trade.pair, stake_amount, trade=trade, is_short=trade.is_short) + self.execute_entry(trade.pair, stake_amount, price=current_rate, + trade=trade, is_short=trade.is_short) if stake_amount is not None and stake_amount < 0.0: # We should decrease our position @@ -586,6 +587,7 @@ class FreqtradeBot(LoggingMixin): ordertype: Optional[str] = None, enter_tag: Optional[str] = None, trade: Optional[Trade] = None, + order_adjust: bool = False ) -> bool: """ Executes a limit buy for the given pair @@ -601,7 +603,7 @@ class FreqtradeBot(LoggingMixin): pos_adjust = trade is not None enter_limit_requested, stake_amount, leverage = self.get_valid_enter_price_and_stake( - pair, price, stake_amount, trade_side, enter_tag, trade) + pair, price, stake_amount, trade_side, enter_tag, trade, order_adjust) if not stake_amount: return False @@ -746,23 +748,26 @@ class FreqtradeBot(LoggingMixin): self, pair: str, price: Optional[float], stake_amount: float, trade_side: LongShort, entry_tag: Optional[str], - trade: Optional[Trade] + trade: Optional[Trade], + order_adjust: bool, ) -> Tuple[float, float, float]: if price: enter_limit_requested = price else: # Calculate price - proposed_enter_rate = self.exchange.get_rate( + enter_limit_requested = self.exchange.get_rate( pair, side='entry', is_short=(trade_side == 'short'), refresh=True) + if not order_adjust: + # Don't call custom_entry_price in order-adjust scenario custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, - default_retval=proposed_enter_rate)( + default_retval=enter_limit_requested)( pair=pair, current_time=datetime.now(timezone.utc), - proposed_rate=proposed_enter_rate, entry_tag=entry_tag, + proposed_rate=enter_limit_requested, entry_tag=entry_tag, side=trade_side, ) - enter_limit_requested = self.get_valid_price(custom_entry_price, proposed_enter_rate) + enter_limit_requested = self.get_valid_price(custom_entry_price, enter_limit_requested) if not enter_limit_requested: raise PricingError('Could not determine entry price.') @@ -1212,7 +1217,8 @@ class FreqtradeBot(LoggingMixin): stake_amount=(order_obj.remaining * order_obj.price), price=adjusted_entry_price, trade=trade, - is_short=trade.is_short + is_short=trade.is_short, + order_adjust=True, ) def cancel_all_open_orders(self) -> None: From a947a1316b005c29b8a5ae58665f6f52ee884bb1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 17:42:01 +0200 Subject: [PATCH 14/35] Add test to ensure stoploss is set properly in live --- tests/test_integration.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 020f77fed..d2ad8c981 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -372,11 +372,15 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, fee, mocker) -> None: freqtrade.enter_positions() assert len(Trade.get_trades().all()) == 1 - trade = Trade.get_trades().first() + trade: Trade = Trade.get_trades().first() assert len(trade.orders) == 1 assert trade.open_order_id is not None assert pytest.approx(trade.stake_amount) == 60 assert trade.open_rate == 1.96 + assert trade.stop_loss_pct is None + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 + assert trade.initial_stop_loss_pct is None # No adjustment freqtrade.process() trade = Trade.get_trades().first() @@ -392,6 +396,10 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, fee, mocker) -> None: assert trade.open_order_id is not None # Open rate is not adjusted yet assert trade.open_rate == 1.96 + assert trade.stop_loss_pct is None + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 + assert trade.initial_stop_loss_pct is None # Fill order mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=True) @@ -401,6 +409,10 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, fee, mocker) -> None: assert trade.open_order_id is None # Open rate is not adjusted yet assert trade.open_rate == 1.99 + assert trade.stop_loss_pct == -0.1 + assert trade.stop_loss == 1.99 * 0.9 + assert trade.initial_stop_loss == 1.99 * 0.9 + assert trade.initial_stop_loss_pct == -0.1 # 2nd order - not filling freqtrade.strategy.adjust_trade_position = MagicMock(return_value=120) From 116b58e97cad2b86aff5e20d97f494b5ef9abd41 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 19:30:42 +0200 Subject: [PATCH 15/35] add "date_minus_candles" method --- freqtrade/exchange/exchange.py | 24 +++++++++++++++++++++--- tests/exchange/test_exchange.py | 17 ++++++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 86f80871b..560da8eb2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2687,9 +2687,10 @@ def timeframe_to_msecs(timeframe: str) -> int: def timeframe_to_prev_date(timeframe: str, date: datetime = None) -> datetime: """ - Use Timeframe and determine last possible candle. + Use Timeframe and determine the candle start date for this date. + Does not round when given a candle start date. :param timeframe: timeframe in string format (e.g. "5m") - :param date: date to use. Defaults to utcnow() + :param date: date to use. Defaults to now(utc) :returns: date of previous candle (with utc timezone) """ if not date: @@ -2704,7 +2705,7 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: """ Use Timeframe and determine next candle. :param timeframe: timeframe in string format (e.g. "5m") - :param date: date to use. Defaults to utcnow() + :param date: date to use. Defaults to now(utc) :returns: date of next candle (with utc timezone) """ if not date: @@ -2714,6 +2715,23 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) +def date_minus_candles( + timeframe: str, candle_count: int, date: Optional[datetime] = None) -> datetime: + """ + subtract X candles from a date. + :param timeframe: timeframe in string format (e.g. "5m") + :param candle_count: Amount of candles to subtract. + :param date: date to use. Defaults to now(utc) + + """ + if not date: + date = datetime.now(timezone.utc) + + tf_min = timeframe_to_minutes(timeframe) + new_date = timeframe_to_prev_date(timeframe, date) - timedelta(minutes=tf_min * candle_count) + return new_date + + def market_is_active(market: Dict) -> bool: """ Return True if the market is active. diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 9d7b77a8e..9dd4e6342 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -17,9 +17,9 @@ from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOr from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff, remove_credentials) -from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs, - timeframe_to_next_date, timeframe_to_prev_date, - timeframe_to_seconds) +from freqtrade.exchange.exchange import (date_minus_candles, market_is_active, timeframe_to_minutes, + timeframe_to_msecs, timeframe_to_next_date, + timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re @@ -3431,6 +3431,17 @@ def test_timeframe_to_next_date(): assert timeframe_to_next_date("5m", date) == date + timedelta(minutes=5) +def test_date_minus_candles(): + + date = datetime(2019, 8, 12, 13, 25, 0, tzinfo=timezone.utc) + + assert date_minus_candles("5m", 3, date) == date - timedelta(minutes=15) + assert date_minus_candles("5m", 5, date) == date - timedelta(minutes=25) + assert date_minus_candles("1m", 6, date) == date - timedelta(minutes=6) + assert date_minus_candles("1h", 3, date) == date - timedelta(hours=3, minutes=25) + assert date_minus_candles("1h", 3) == timeframe_to_prev_date('1h') - timedelta(hours=3) + + @pytest.mark.parametrize( "market_symbol,base,quote,exchange,spot,margin,futures,trademode,add_dict,expected_result", [ From d60d0f64d209d1013e2d32938f72bc8a94598af8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 May 2022 19:32:28 +0200 Subject: [PATCH 16/35] Revert ohlcv_candle_limit logic for okx --- freqtrade/exchange/exchange.py | 5 ++++- freqtrade/exchange/okx.py | 18 ++++++++---------- tests/exchange/test_okx.py | 3 +++ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 560da8eb2..57a7f2086 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -619,7 +619,10 @@ class Exchange: Checks if required startup_candles is more than ohlcv_candle_limit(). Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. """ - candle_limit = self.ohlcv_candle_limit(timeframe, self._config['candle_type_def'], None) + + candle_limit = self.ohlcv_candle_limit( + timeframe, self._config['candle_type_def'], + date_minus_candles(timeframe, startup_candles)) # Require one more candle - to account for the still open candle. candle_count = startup_candles + 1 # Allow 5 calls to the exchange per pair diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index c8324e62e..ad41984e7 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -10,7 +10,7 @@ from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier -from freqtrade.exchange.exchange import timeframe_to_minutes +from freqtrade.exchange.exchange import date_minus_candles logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ class Okx(Exchange): """ _ft_has: Dict = { - "ohlcv_candle_limit": 300, # Warning, special case with data prior to X months + "ohlcv_candle_limit": 100, # Warning, special case with data prior to X months "mark_ohlcv_timeframe": "4h", "funding_fee_timeframe": "8h", } @@ -53,15 +53,13 @@ class Okx(Exchange): :param since_ms: Candle-type :return: Candle limit as integer """ - now = datetime.now(timezone.utc) - offset_mins = timeframe_to_minutes(timeframe) * self._ft_has['ohlcv_candle_limit'] - if since_ms and since_ms < ((now - timedelta(minutes=offset_mins)).timestamp() * 1000): - return 100 - if candle_type not in (CandleType.FUTURES, CandleType.SPOT): - return 100 + if ( + candle_type in (CandleType.FUTURES, CandleType.SPOT) and + (not since_ms or since_ms > (date_minus_candles(timeframe, 300).timestamp() * 1000)) + ): + return 300 - return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( - timeframe, self._ft_has.get('ohlcv_candle_limit'))) + return super().ohlcv_candle_limit(timeframe, candle_type, since_ms) @retrier def additional_exchange_init(self) -> None: diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 2804d471a..19c09ad9e 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -20,14 +20,17 @@ def test_okx_ohlcv_candle_limit(default_conf, mocker): assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES) == 300 assert exchange.ohlcv_candle_limit(timeframe, CandleType.MARK) == 100 assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUNDING_RATE) == 100 + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, start_time) == 100 assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, start_time) == 100 assert exchange.ohlcv_candle_limit(timeframe, CandleType.MARK, start_time) == 100 assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUNDING_RATE, start_time) == 100 one_call = int((datetime.now(timezone.utc) - timedelta( minutes=290 * timeframe_to_minutes(timeframe))).timestamp() * 1000) + assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, one_call) == 300 assert exchange.ohlcv_candle_limit(timeframe, CandleType.FUTURES, one_call) == 300 + one_call = int((datetime.now(timezone.utc) - timedelta( minutes=320 * timeframe_to_minutes(timeframe))).timestamp() * 1000) assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT, one_call) == 100 From 9143e9ecb15c9756b6e0f4a7f437a15cddc12385 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 15:12:29 +0200 Subject: [PATCH 17/35] Add some safety measures for new startup_candles verification --- freqtrade/exchange/exchange.py | 3 ++- freqtrade/exchange/okx.py | 1 - tests/exchange/test_exchange.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 57a7f2086..a07ea3596 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -622,7 +622,8 @@ class Exchange: candle_limit = self.ohlcv_candle_limit( timeframe, self._config['candle_type_def'], - date_minus_candles(timeframe, startup_candles)) + int(date_minus_candles(timeframe, startup_candles).timestamp() * 1000) + if timeframe else None) # Require one more candle - to account for the still open candle. candle_count = startup_candles + 1 # Allow 5 calls to the exchange per pair diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index ad41984e7..c0431c7fc 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional, Tuple import ccxt diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 9dd4e6342..e580c82d3 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -939,6 +939,7 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker): def test_validate_timeframes_not_in_config(default_conf, mocker): + # TODO: this test does not assert ... del default_conf["timeframe"] api_mock = MagicMock() id_mock = PropertyMock(return_value='test_exchange') @@ -954,6 +955,7 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') mocker.patch('freqtrade.exchange.Exchange.validate_pricing') + mocker.patch('freqtrade.exchange.Exchange.validate_required_startup_candles') Exchange(default_conf) From 18fd3bb3332f2d50401fa6b428d073615b02db7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 15:30:57 +0200 Subject: [PATCH 18/35] Update stoploss handling for entry-order adjustment --- freqtrade/optimize/backtesting.py | 6 +----- freqtrade/persistence/trade_model.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f439e4e63..621812b0a 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -780,8 +780,6 @@ class Backtesting: # interest_rate=interest_rate, orders=[], ) - elif trade.nr_of_successful_entries == 0: - trade.open_rate = propose_rate trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) @@ -814,11 +812,11 @@ class Backtesting: remaining=amount, cost=stake_amount + trade.fee_open, ) + trade.orders.append(order) if pos_adjust and self._get_order_filled(order.price, row): order.close_bt_order(current_time, trade) else: trade.open_order_id = str(self.order_id_counter) - trade.orders.append(order) trade.recalc_trade_from_orders() return trade @@ -942,8 +940,6 @@ class Backtesting: requested_rate=requested_rate, requested_stake=(order.remaining * order.price), direction='short' if trade.is_short else 'long') - trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, - initial=False, refresh=True) else: # assumption: there can't be multiple open entry orders at any given time return (trade.nr_of_successful_entries == 0) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index bbdeeef47..358e776e3 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -153,6 +153,7 @@ class Order(_DECL_BASE): and len(trade.select_filled_orders(trade.entry_side)) == 1): trade.open_rate = self.price trade.recalc_open_trade_value() + trade.adjust_stop_loss(trade.open_rate, trade.stop_loss_pct, refresh=True) @staticmethod def update_orders(orders: List['Order'], order: Dict[str, Any]): @@ -502,7 +503,7 @@ class LocalTrade(): if initial and not (self.stop_loss is None or self.stop_loss == 0): # Don't modify if called with initial and nothing to do return - refresh = False if self.nr_of_successful_entries > 0 else refresh + refresh = True if refresh and self.nr_of_successful_entries == 1 else False leverage = self.leverage or 1.0 if self.is_short: From 706994340f36e5336e35afe4a580015992e131b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 17:06:40 +0200 Subject: [PATCH 19/35] Fix bad docstring --- freqtrade/exchange/exchange.py | 2 +- freqtrade/exchange/okx.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a07ea3596..ee804aa68 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -317,7 +317,7 @@ class Exchange: per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit :param timeframe: Timeframe to check :param candle_type: Candle-type - :param since_ms: Candle-type + :param since_ms: Starting timestamp :return: Candle limit as integer """ return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index c0431c7fc..012f51080 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -49,7 +49,7 @@ class Okx(Exchange): * 100 candles for additional candles (not futures or spot). :param timeframe: Timeframe to check :param candle_type: Candle-type - :param since_ms: Candle-type + :param since_ms: Starting timestamp :return: Candle limit as integer """ if ( From a8f064a8cb4b84914820c98bc895ff6c7a0dda71 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 17:33:00 +0200 Subject: [PATCH 20/35] Fix exit_reason assignment in live mode --- freqtrade/freqtradebot.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 07b055309..315db3ae6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1414,6 +1414,7 @@ class FreqtradeBot(LoggingMixin): open_date=trade.open_date_utc, ) exit_type = 'exit' + exit_reason = exit_tag or exit_check.exit_reason if exit_check.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS): exit_type = 'stoploss' @@ -1431,7 +1432,7 @@ class FreqtradeBot(LoggingMixin): pair=trade.pair, trade=trade, current_time=datetime.now(timezone.utc), proposed_rate=proposed_limit_rate, current_profit=current_profit, - exit_tag=exit_check.exit_reason) + exit_tag=exit_reason) limit = self.get_valid_price(custom_exit_price, proposed_limit_rate) @@ -1448,8 +1449,8 @@ class FreqtradeBot(LoggingMixin): if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, - time_in_force=time_in_force, exit_reason=exit_check.exit_reason, - sell_reason=exit_check.exit_reason, # sellreason -> compatibility + time_in_force=time_in_force, exit_reason=exit_reason, + sell_reason=exit_reason, # sellreason -> compatibility current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of exiting {trade.pair}") return False @@ -1478,7 +1479,7 @@ class FreqtradeBot(LoggingMixin): trade.open_order_id = order['id'] trade.exit_order_status = '' trade.close_rate_requested = limit - trade.exit_reason = exit_tag or exit_check.exit_reason + trade.exit_reason = exit_reason # Lock pair for one candle to prevent immediate re-trading self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), From a0b25938f472d4941b7f08986a88c0c69b356e7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 17:41:50 +0200 Subject: [PATCH 21/35] Fix exit_reason assignment in backtesting --- freqtrade/optimize/backtesting.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 621812b0a..64107ae18 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -535,6 +535,7 @@ class Backtesting: if exit_.exit_flag: trade.close_date = exit_candle_time + exit_reason = exit_.exit_reason trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) try: @@ -545,6 +546,15 @@ class Backtesting: current_profit = trade.calc_profit_ratio(closerate) order_type = self.strategy.order_types['exit'] if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT): + # Checks and adds an exit tag, after checking that the length of the + # row has the length for an exit tag column + if( + len(row) > EXIT_TAG_IDX + and row[EXIT_TAG_IDX] is not None + and len(row[EXIT_TAG_IDX]) > 0 + and exit_.exit_type in (ExitType.EXIT_SIGNAL,) + ): + exit_reason = row[EXIT_TAG_IDX] # Custom exit pricing only for exit-signals if order_type == 'limit': closerate = strategy_safe_wrapper(self.strategy.custom_exit_price, @@ -552,7 +562,7 @@ class Backtesting: pair=trade.pair, trade=trade, current_time=exit_candle_time, proposed_rate=closerate, current_profit=current_profit, - exit_tag=exit_.exit_reason) + exit_tag=exit_reason) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately if trade.is_short: @@ -566,22 +576,12 @@ class Backtesting: pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount, rate=closerate, time_in_force=time_in_force, - sell_reason=exit_.exit_reason, # deprecated - exit_reason=exit_.exit_reason, + sell_reason=exit_reason, # deprecated + exit_reason=exit_reason, current_time=exit_candle_time): return None - trade.exit_reason = exit_.exit_reason - - # Checks and adds an exit tag, after checking that the length of the - # row has the length for an exit tag column - if( - len(row) > EXIT_TAG_IDX - and row[EXIT_TAG_IDX] is not None - and len(row[EXIT_TAG_IDX]) > 0 - and exit_.exit_type in (ExitType.EXIT_SIGNAL,) - ): - trade.exit_reason = row[EXIT_TAG_IDX] + trade.exit_reason = exit_reason self.order_id_counter += 1 order = Order( From 86af3fe0e7766a3dd86701d8142aeb327bafd7d2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 19:22:12 +0200 Subject: [PATCH 22/35] Update image versions from 3.9 to 3.10 --- .github/workflows/ci.yml | 6 +++--- Dockerfile | 2 +- docker/Dockerfile.armhf | 2 +- setup.cfg | 2 +- setup.sh | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96575f034..09946e6b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -273,7 +273,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: 3.10 - name: pre-commit dependencies run: | @@ -292,7 +292,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: 3.10 - name: Documentation build run: | @@ -358,7 +358,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.9 - name: Extract branch name shell: bash diff --git a/Dockerfile b/Dockerfile index 8f5b85698..5f7b52265 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.9-slim-bullseye as base +FROM python:3.10.4-slim-bullseye as base # Setup env ENV LANG C.UTF-8 diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index 16f2aebcd..73fc681eb 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM python:3.9.9-slim-bullseye as base +FROM python:3.9.12-slim-bullseye as base # Setup env ENV LANG C.UTF-8 diff --git a/setup.cfg b/setup.cfg index edbd320c3..042517ec9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ tests_require = pytest-mock packages = find: -python_requires = >=3.6 +python_requires = >=3.8 [options.entry_points] console_scripts = diff --git a/setup.sh b/setup.sh index dcf6c02c7..bb51c3a2f 100755 --- a/setup.sh +++ b/setup.sh @@ -25,7 +25,7 @@ function check_installed_python() { exit 2 fi - for v in 9 10 8 + for v in 10 9 8 do PYTHON="python3.${v}" which $PYTHON From 008ee148890b3c396c311483340bfb698ebd925a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 May 2022 19:25:27 +0200 Subject: [PATCH 23/35] Improve ci to run on ubuntu 22.04 --- .github/workflows/ci.yml | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09946e6b5..d11285ba4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-18.04, ubuntu-20.04 ] + os: [ ubuntu-18.04, ubuntu-20.04, ubuntu-22.04 ] python-version: ["3.8", "3.9", "3.10"] steps: @@ -70,7 +70,7 @@ jobs: if: matrix.python-version == '3.9' - name: Coveralls - if: (runner.os == 'Linux' && matrix.python-version == '3.8') + if: (runner.os == 'Linux' && matrix.python-version == '3.9') env: # Coveralls token. Not used as secret due to github not providing secrets to forked repositories COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu @@ -157,24 +157,9 @@ jobs: pip install -e . - name: Tests - if: (runner.os != 'Linux' || matrix.python-version != '3.8') run: | pytest --random-order - - name: Tests (with cov) - if: (runner.os == 'Linux' && matrix.python-version == '3.8') - run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc - - - name: Coveralls - if: (runner.os == 'Linux' && matrix.python-version == '3.8') - env: - # Coveralls token. Not used as secret due to github not providing secrets to forked repositories - COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu - run: | - # Allow failure for coveralls - coveralls -v || true - - name: Backtesting run: | cp config_examples/config_bittrex.example.json config.json @@ -273,7 +258,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.10 + python-version: "3.10" - name: pre-commit dependencies run: | @@ -292,7 +277,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.10 + python-version: "3.10" - name: Documentation build run: | @@ -358,7 +343,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: "3.9" - name: Extract branch name shell: bash From e21f6a7787091e2c3831f872bf26aaed3df0184c Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 16 May 2022 07:28:40 +0900 Subject: [PATCH 24/35] missing newline --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 1a9be4503..259d2c831 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1417,7 +1417,7 @@ class Telegram(RPCHandler): "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" "*/forceexit |all:* `Instantly exits the given trade or all trades, " "regardless of profit`\n" - "*/fe |all:* `Alias to /forceexit`" + "*/fe |all:* `Alias to /forceexit`\n" f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}" "*/delete :* `Instantly delete the given trade in the database`\n" "*/whitelist:* `Show current whitelist` \n" From 2cb8eecf18eed273df41ee2702e54def89831b03 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 16 May 2022 07:43:36 +0900 Subject: [PATCH 25/35] add space --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 259d2c831..f26de8b5c 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1410,7 +1410,7 @@ class Telegram(RPCHandler): "Optionally takes a rate at which to sell " "(only applies to limit orders).` \n") message = ( - "_BotControl_\n" + "_Bot Control_\n" "------------\n" "*/start:* `Starts the trader`\n" "*/stop:* Stops the trader\n" From 4fc6857d8778cb423ab228750a8edb23c2cfe0c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:25 +0000 Subject: [PATCH 26/35] Bump time-machine from 2.6.0 to 2.7.0 Bumps [time-machine](https://github.com/adamchainz/time-machine) from 2.6.0 to 2.7.0. - [Release notes](https://github.com/adamchainz/time-machine/releases) - [Changelog](https://github.com/adamchainz/time-machine/blob/main/HISTORY.rst) - [Commits](https://github.com/adamchainz/time-machine/compare/2.6.0...2.7.0) --- updated-dependencies: - dependency-name: time-machine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 60f4da1a7..58eaef3e2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,7 +16,7 @@ pytest-mock==3.7.0 pytest-random-order==1.0.4 isort==5.10.1 # For datetime mocking -time-machine==2.6.0 +time-machine==2.7.0 # Convert jupyter notebooks to markdown documents nbconvert==6.5.0 From 47c116a423cabe9a0444c29be9edef465af86ecb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:28 +0000 Subject: [PATCH 27/35] Bump fastapi from 0.76.0 to 0.78.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.76.0 to 0.78.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.76.0...0.78.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bed3a7fed..14edbe16c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ orjson==3.6.8 sdnotify==0.3.2 # API Server -fastapi==0.76.0 +fastapi==0.78.0 uvicorn==0.17.6 pyjwt==2.3.0 aiofiles==0.8.0 From 748055892cbc0f32a6610960ec0e58521e6b45fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:33 +0000 Subject: [PATCH 28/35] Bump plotly from 5.7.0 to 5.8.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.7.0 to 5.8.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.7.0...v5.8.0) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index d9faed301..e17efbc71 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.7.0 +plotly==5.8.0 From 9fc21686ede5c7f28f978a9198535307a2a92b14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:40 +0000 Subject: [PATCH 29/35] Bump flake8-tidy-imports from 4.7.0 to 4.8.0 Bumps [flake8-tidy-imports](https://github.com/adamchainz/flake8-tidy-imports) from 4.7.0 to 4.8.0. - [Release notes](https://github.com/adamchainz/flake8-tidy-imports/releases) - [Changelog](https://github.com/adamchainz/flake8-tidy-imports/blob/main/HISTORY.rst) - [Commits](https://github.com/adamchainz/flake8-tidy-imports/compare/4.7.0...4.8.0) --- updated-dependencies: - dependency-name: flake8-tidy-imports dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 60f4da1a7..17a505912 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ coveralls==3.3.1 flake8==4.0.1 -flake8-tidy-imports==4.7.0 +flake8-tidy-imports==4.8.0 mypy==0.950 pre-commit==2.19.0 pytest==7.1.2 From 9e44d697746369fa6467e053698a698365069cfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:53 +0000 Subject: [PATCH 30/35] Bump ccxt from 1.81.81 to 1.82.61 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.81.81 to 1.82.61. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.81.81...1.82.61) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bed3a7fed..cdd72403d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.3 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.81.81 +ccxt==1.82.61 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From a8b4066f85fdb08c254e06be91a3818e62dc1855 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:01:57 +0000 Subject: [PATCH 31/35] Bump mkdocs-material from 8.2.14 to 8.2.15 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.2.14 to 8.2.15. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.2.14...8.2.15) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index b26e448ea..3fa35d80d 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.2.14 +mkdocs-material==8.2.15 mdx_truly_sane_lists==1.2 pymdown-extensions==9.4 jinja2==3.1.2 From dd1b84f938db950821d4d40e14be83e1b0faff5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 03:02:00 +0000 Subject: [PATCH 32/35] Bump filelock from 3.6.0 to 3.7.0 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.6.0 to 3.7.0. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/py-filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.6.0...3.7.0) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 32fc3f4b9..17a7c7b8c 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,5 +5,5 @@ scipy==1.8.0 scikit-learn==1.0.2 scikit-optimize==0.9.0 -filelock==3.6.0 +filelock==3.7.0 progressbar2==4.0.0 From bd65236e17f53ed19abf6a687ff0cd8d856d838f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 04:37:25 +0000 Subject: [PATCH 33/35] Bump pyjwt from 2.3.0 to 2.4.0 Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.3.0...2.4.0) --- updated-dependencies: - dependency-name: pyjwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 95826b71c..90ddcd1b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ sdnotify==0.3.2 # API Server fastapi==0.78.0 uvicorn==0.17.6 -pyjwt==2.3.0 +pyjwt==2.4.0 aiofiles==0.8.0 psutil==5.9.0 From f5183df0f1c6e41fb3b8d39fca18b8f829bf73d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 04:37:41 +0000 Subject: [PATCH 34/35] Bump scikit-learn from 1.0.2 to 1.1.0 Bumps [scikit-learn](https://github.com/scikit-learn/scikit-learn) from 1.0.2 to 1.1.0. - [Release notes](https://github.com/scikit-learn/scikit-learn/releases) - [Commits](https://github.com/scikit-learn/scikit-learn/compare/1.0.2...1.1.0) --- updated-dependencies: - dependency-name: scikit-learn dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 17a7c7b8c..0b91636f1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -3,7 +3,7 @@ # Required for hyperopt scipy==1.8.0 -scikit-learn==1.0.2 +scikit-learn==1.1.0 scikit-optimize==0.9.0 filelock==3.7.0 progressbar2==4.0.0 From 528509f809b2474218e24ac5442d889b4ca1fce9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 May 2022 19:18:13 +0200 Subject: [PATCH 35/35] Extract get_price_side from get_rate --- freqtrade/exchange/exchange.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ee804aa68..156216557 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1457,6 +1457,23 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def _get_price_side(self, side: str, is_short: bool, conf_strategy: Dict) -> str: + price_side = conf_strategy['price_side'] + + if price_side in ('same', 'other'): + price_map = { + ('entry', 'long', 'same'): 'bid', + ('entry', 'long', 'other'): 'ask', + ('entry', 'short', 'same'): 'ask', + ('entry', 'short', 'other'): 'bid', + ('exit', 'long', 'same'): 'ask', + ('exit', 'long', 'other'): 'bid', + ('exit', 'short', 'same'): 'bid', + ('exit', 'short', 'other'): 'ask', + } + price_side = price_map[(side, 'short' if is_short else 'long', price_side)] + return price_side + def get_rate(self, pair: str, refresh: bool, side: EntryExit, is_short: bool) -> float: """ @@ -1483,20 +1500,7 @@ class Exchange: conf_strategy = self._config.get(strat_name, {}) - price_side = conf_strategy['price_side'] - - if price_side in ('same', 'other'): - price_map = { - ('entry', 'long', 'same'): 'bid', - ('entry', 'long', 'other'): 'ask', - ('entry', 'short', 'same'): 'ask', - ('entry', 'short', 'other'): 'bid', - ('exit', 'long', 'same'): 'ask', - ('exit', 'long', 'other'): 'bid', - ('exit', 'short', 'same'): 'bid', - ('exit', 'short', 'other'): 'ask', - } - price_side = price_map[(side, 'short' if is_short else 'long', price_side)] + price_side = self._get_price_side(side, is_short, conf_strategy) price_side_word = price_side.capitalize()