From 6ff3b178b0916ac726f65d3ad2d5929c44b5292f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Apr 2022 15:26:58 +0200 Subject: [PATCH 01/26] Add direction column to pairlocks --- freqtrade/persistence/migrations.py | 57 ++++++++++++++++++++++++----- freqtrade/persistence/models.py | 2 + tests/test_persistence.py | 47 ++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 9 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index f020f990c..eff2d69f3 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -9,7 +9,7 @@ from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) -def get_table_names_for_table(inspector, tabletype): +def get_table_names_for_table(inspector, tabletype) -> List[str]: return [t for t in inspector.get_table_names() if t.startswith(tabletype)] @@ -21,7 +21,7 @@ def get_column_def(columns: List, column: str, default: str) -> str: return default if not has_column(columns, column) else column -def get_backup_name(tabs, backup_prefix: str): +def get_backup_name(tabs: List[str], backup_prefix: str): table_back_name = backup_prefix for i, table_back_name in enumerate(tabs): table_back_name = f'{backup_prefix}{i}' @@ -56,6 +56,16 @@ def set_sequence_ids(engine, order_id, trade_id): connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}")) +def drop_index_on_table(engine, inspector, table_bak_name): + with engine.begin() as connection: + # drop indexes on backup table in new session + for index in inspector.get_indexes(table_bak_name): + if engine.name == 'mysql': + connection.execute(text(f"drop index {index['name']} on {table_bak_name}")) + else: + connection.execute(text(f"drop index {index['name']}")) + + def migrate_trades_and_orders_table( decl_base, inspector, engine, trade_back_name: str, cols: List, @@ -116,13 +126,7 @@ def migrate_trades_and_orders_table( with engine.begin() as connection: connection.execute(text(f"alter table trades rename to {trade_back_name}")) - with engine.begin() as connection: - # drop indexes on backup table in new session - for index in inspector.get_indexes(trade_back_name): - if engine.name == 'mysql': - connection.execute(text(f"drop index {index['name']} on {trade_back_name}")) - else: - connection.execute(text(f"drop index {index['name']}")) + drop_index_on_table(engine, inspector, trade_back_name) order_id, trade_id = get_last_sequence_ids(engine, trade_back_name, order_back_name) @@ -205,6 +209,31 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): """)) +def migrate_pairlocks_table( + decl_base, inspector, engine, + pairlock_back_name: str, cols: List): + + # Schema migration necessary + with engine.begin() as connection: + connection.execute(text(f"alter table pairlocks rename to {pairlock_back_name}")) + + drop_index_on_table(engine, inspector, pairlock_back_name) + + direction = get_column_def(cols, 'direction', "'*'") + + # let SQLAlchemy create the schema as required + decl_base.metadata.create_all(engine) + # Copy data back - following the correct schema + with engine.begin() as connection: + connection.execute(text(f"""insert into pairlocks + (id, pair, direction, reason, lock_time, + lock_end_time, active) + select id, pair, {direction} direction, reason, lock_time, + lock_end_time, active + from {pairlock_back_name} + """)) + + def set_sqlite_to_wal(engine): if engine.name == 'sqlite' and str(engine.url) != 'sqlite://': # Set Mode to @@ -220,10 +249,13 @@ def check_migrate(engine, decl_base, previous_tables) -> None: cols_trades = inspector.get_columns('trades') cols_orders = inspector.get_columns('orders') + cols_pairlocks = inspector.get_columns('pairlocks') tabs = get_table_names_for_table(inspector, 'trades') table_back_name = get_backup_name(tabs, 'trades_bak') order_tabs = get_table_names_for_table(inspector, 'orders') order_table_bak_name = get_backup_name(order_tabs, 'orders_bak') + pairlock_tabs = get_table_names_for_table(inspector, 'pairlocks') + pairlock_table_bak_name = get_backup_name(pairlock_tabs, 'pairlocks_bak') # Check if migration necessary # Migrates both trades and orders table! @@ -236,6 +268,13 @@ def check_migrate(engine, decl_base, previous_tables) -> None: decl_base, inspector, engine, table_back_name, cols_trades, order_table_bak_name, cols_orders) + if not has_column(cols_pairlocks, 'direction'): + logger.info(f"Running database migration for pairlocks - " + f"backup: {pairlock_table_bak_name}") + + migrate_pairlocks_table( + decl_base, inspector, engine, pairlock_table_bak_name, cols_pairlocks + ) if 'orders' not in previous_tables and 'trades' in previous_tables: raise OperationalException( "Your database seems to be very old. " diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index a9c07f12c..4aa1c6a4d 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -1428,6 +1428,8 @@ class PairLock(_DECL_BASE): id = Column(Integer, primary_key=True) pair = Column(String(25), nullable=False, index=True) + # lock direction - long, short or * (for both) + direction = Column(String(25), nullable=False, default="*") reason = Column(String(255), nullable=True) # Time the pair was locked (start time) lock_time = Column(DateTime, nullable=False) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 801e0e35f..58d3a4de4 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -15,6 +15,7 @@ from freqtrade.enums import TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.persistence import LocalTrade, Order, Trade, clean_dry_run_db, init_db from freqtrade.persistence.migrations import get_last_sequence_ids, set_sequence_ids +from freqtrade.persistence.models import PairLock from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re @@ -1427,6 +1428,52 @@ def test_migrate_set_sequence_ids(): assert engine.begin.call_count == 0 +def test_migrate_pairlocks(mocker, default_conf, fee, caplog): + """ + Test Database migration (starting with new pairformat) + """ + caplog.set_level(logging.DEBUG) + # Always create all columns apart from the last! + create_table_old = """CREATE TABLE pairlocks ( + id INTEGER NOT NULL, + pair VARCHAR(25) NOT NULL, + reason VARCHAR(255), + lock_time DATETIME NOT NULL, + lock_end_time DATETIME NOT NULL, + active BOOLEAN NOT NULL, + PRIMARY KEY (id) + ) + """ + create_index1 = "CREATE INDEX ix_pairlocks_pair ON pairlocks (pair)" + create_index2 = "CREATE INDEX ix_pairlocks_lock_end_time ON pairlocks (lock_end_time)" + create_index3 = "CREATE INDEX ix_pairlocks_active ON pairlocks (active)" + insert_table_old = """INSERT INTO pairlocks ( + id, pair, reason, lock_time, lock_end_time, active) + VALUES (1, 'ETH/BTC', 'Auto lock', '2021-07-12 18:41:03', '2021-07-11 18:45:00', 1) + """ + insert_table_old2 = """INSERT INTO pairlocks ( + id, pair, reason, lock_time, lock_end_time, active) + VALUES (2, '*', 'Lock all', '2021-07-12 18:41:03', '2021-07-12 19:00:00', 1) + """ + engine = create_engine('sqlite://') + mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) + # Create table using the old format + with engine.begin() as connection: + connection.execute(text(create_table_old)) + + connection.execute(text(insert_table_old)) + connection.execute(text(insert_table_old2)) + connection.execute(text(create_index1)) + connection.execute(text(create_index2)) + connection.execute(text(create_index3)) + + init_db(default_conf['db_url'], default_conf['dry_run']) + + assert len(PairLock.query.all()) == 2 + assert len(PairLock.query.filter(PairLock.pair == '*').all()) == 1 + assert len(PairLock.query.filter(PairLock.pair == 'ETH/BTC').all()) == 1 + + def test_adjust_stop_loss(fee): trade = Trade( pair='ADA/USDT', From 9e199165b4957624d13e8fbda9bfc6bdd28d3d83 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Apr 2022 19:58:20 +0200 Subject: [PATCH 02/26] Update protection-interface to support per-side locks --- docs/includes/protections.md | 5 +++- freqtrade/freqtradebot.py | 8 +++---- freqtrade/optimize/backtesting.py | 9 +++---- freqtrade/plugins/protectionmanager.py | 11 +++++---- .../plugins/protections/cooldown_period.py | 12 +++++----- freqtrade/plugins/protections/iprotection.py | 7 +++--- .../plugins/protections/low_profit_pairs.py | 12 +++++----- .../protections/max_drawdown_protection.py | 14 +++++------ .../plugins/protections/stoploss_guard.py | 24 ++++++++++++------- 9 files changed, 58 insertions(+), 44 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 0757d2f6d..a242a6256 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -48,6 +48,8 @@ If `trade_limit` or more trades resulted in stoploss, trading will stop for `sto This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. +Similarly, this protection will by default look at all trades (long and short). For futures bots, setting `only_per_side` will make the bot only one side, and will then only lock this one side. + The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. ``` python @@ -59,7 +61,8 @@ def protections(self): "lookback_period_candles": 24, "trade_limit": 4, "stop_duration_candles": 4, - "only_per_pair": False + "only_per_pair": False, + "only_per_side": True } ] ``` diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 57d7cac3c..d3408ada2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1604,21 +1604,21 @@ class FreqtradeBot(LoggingMixin): if not trade.is_open: if send_msg and not stoploss_order and not trade.open_order_id: self._notify_exit(trade, '', True) - self.handle_protections(trade.pair) + self.handle_protections(trade.pair, trade.trade_direction) elif send_msg and not trade.open_order_id: # Enter fill self._notify_enter(trade, order, fill=True) return False - def handle_protections(self, pair: str) -> None: - prot_trig = self.protections.stop_per_pair(pair) + def handle_protections(self, pair: str, side: str) -> None: + prot_trig = self.protections.stop_per_pair(pair, side=side) if prot_trig: msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } msg.update(prot_trig.to_json()) self.rpc.send_msg(msg) - prot_trig_glb = self.protections.global_stop() + prot_trig_glb = self.protections.global_stop(side=side) if prot_trig_glb: msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, } msg.update(prot_trig_glb.to_json()) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5442e425b..86c52e737 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -849,10 +849,10 @@ class Backtesting: return 'short' return None - def run_protections(self, enable_protections, pair: str, current_time: datetime): + def run_protections(self, enable_protections, pair: str, current_time: datetime, side: str): if enable_protections: - self.protections.stop_per_pair(pair, current_time) - self.protections.global_stop(current_time) + self.protections.stop_per_pair(pair, current_time, side) + self.protections.global_stop(current_time, side) def check_order_cancel(self, trade: LocalTrade, current_time) -> bool: """ @@ -1002,7 +1002,8 @@ class Backtesting: LocalTrade.close_bt_trade(trade) trades.append(trade) self.wallets.update() - self.run_protections(enable_protections, pair, current_time) + self.run_protections( + enable_protections, pair, current_time, trade.trade_direction) # Move time one configured time_interval ahead. self.progress.increment() diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 2510d6fee..e8c3fa02d 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -44,13 +44,14 @@ class ProtectionManager(): """ return [{p.name: p.short_desc()} for p in self._protection_handlers] - def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]: + def global_stop(self, now: Optional[datetime] = None, side: str = 'long') -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) result = None for protection_handler in self._protection_handlers: if protection_handler.has_global_stop: - lock, until, reason = protection_handler.global_stop(now) + lock, until, reason, lock_side = protection_handler.global_stop( + date_now=now, side=side) # Early stopping - first positive result blocks further trades if lock and until: @@ -58,13 +59,15 @@ class ProtectionManager(): result = PairLocks.lock_pair('*', until, reason, now=now) return result - def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]: + def stop_per_pair( + self, pair, now: Optional[datetime] = None, side: str = 'long') -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) result = None for protection_handler in self._protection_handlers: if protection_handler.has_local_stop: - lock, until, reason = protection_handler.stop_per_pair(pair, now) + lock, until, reason, lock_side = protection_handler.stop_per_pair( + pair=pair, date_now=now, side=side) if lock and until: if not PairLocks.is_pair_locked(pair, until): result = PairLocks.lock_pair(pair, until, reason, now=now) diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index a2d8eca34..a75e4fc67 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -26,7 +26,7 @@ class CooldownPeriod(IProtection): """ return (f"{self.name} - Cooldown period of {self.stop_duration_str}.") - def _cooldown_period(self, pair: str, date_now: datetime, ) -> ProtectionReturn: + def _cooldown_period(self, pair: str, date_now: datetime) -> ProtectionReturn: """ Get last trade for this pair """ @@ -45,11 +45,11 @@ class CooldownPeriod(IProtection): self.log_once(f"Cooldown for {pair} for {self.stop_duration_str}.", logger.info) until = self.calculate_lock_end([trade], self._stop_duration) - return True, until, self._reason() + return True, until, self._reason(), None - return False, None, None + return False, None, None, None - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -57,9 +57,9 @@ class CooldownPeriod(IProtection): If true, all pairs will be locked with until """ # Not implemented for cooldown period. - return False, None, None + return False, None, None, None - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index e0a89e334..5f1029eb5 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -12,7 +12,8 @@ from freqtrade.persistence import LocalTrade logger = logging.getLogger(__name__) -ProtectionReturn = Tuple[bool, Optional[datetime], Optional[str]] +# lock, until, reason, lock_side +ProtectionReturn = Tuple[bool, Optional[datetime], Optional[str], Optional[str]] class IProtection(LoggingMixin, ABC): @@ -80,14 +81,14 @@ class IProtection(LoggingMixin, ABC): """ @abstractmethod - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". """ @abstractmethod - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 7822ce73c..38fd6e734 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -51,7 +51,7 @@ class LowProfitPairs(IProtection): # trades = Trade.get_trades(filters).all() if len(trades) < self._trade_limit: # Not enough trades in the relevant period - return False, None, None + return False, None, None, None profit = sum(trade.close_profit for trade in trades if trade.close_profit) if profit < self._required_profit: @@ -60,20 +60,20 @@ class LowProfitPairs(IProtection): f"within {self._lookback_period} minutes.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason(profit) + return True, until, self._reason(profit), None - return False, None, None + return False, None, None, None - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". :return: Tuple of [bool, until, reason]. If true, all pairs will be locked with until """ - return False, None, None + return False, None, None, None - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index b6ef92bd5..e6cc2ba79 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -51,14 +51,14 @@ class MaxDrawdown(IProtection): if len(trades) < self._trade_limit: # Not enough trades in the relevant period - return False, None, None + return False, None, None, None # Drawdown is always positive try: # TODO: This should use absolute profit calculation, considering account balance. drawdown, _, _, _, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') except ValueError: - return False, None, None + return False, None, None, None if drawdown > self._max_allowed_drawdown: self.log_once( @@ -66,11 +66,11 @@ class MaxDrawdown(IProtection): f" within {self.lookback_period_str}.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason(drawdown) + return True, until, self._reason(drawdown), None - return False, None, None + return False, None, None, None - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -79,11 +79,11 @@ class MaxDrawdown(IProtection): """ return self._max_drawdown(date_now) - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". :return: Tuple of [bool, until, reason]. If true, this pair will be locked with until """ - return False, None, None + return False, None, None, None diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 8d7fb2a0e..c8e4dcd21 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any, Dict, Optional from freqtrade.enums import ExitType from freqtrade.persistence import Trade @@ -21,6 +21,7 @@ class StoplossGuard(IProtection): self._trade_limit = protection_config.get('trade_limit', 10) self._disable_global_stop = protection_config.get('only_per_pair', False) + self._only_per_side = protection_config.get('only_per_side', False) def short_desc(self) -> str: """ @@ -36,7 +37,8 @@ class StoplossGuard(IProtection): return (f'{self._trade_limit} stoplosses in {self._lookback_period} min, ' f'locking for {self._stop_duration} min.') - def _stoploss_guard(self, date_now: datetime, pair: str = None) -> ProtectionReturn: + def _stoploss_guard( + self, date_now: datetime, pair: Optional[str], side: str) -> ProtectionReturn: """ Evaluate recent trades """ @@ -48,15 +50,19 @@ class StoplossGuard(IProtection): ExitType.STOPLOSS_ON_EXCHANGE.value) and trade.close_profit and trade.close_profit < 0)] + if self._only_per_side and side: + # Long or short trades only + trades = [trade for trade in trades if trade.trade_direction == side] + if len(trades) < self._trade_limit: - return False, None, None + return False, None, None, None self.log_once(f"Trading stopped due to {self._trade_limit} " f"stoplosses within {self._lookback_period} minutes.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason() + return True, until, self._reason(), (side if self._only_per_side else None) - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -64,14 +70,14 @@ class StoplossGuard(IProtection): If true, all pairs will be locked with until """ if self._disable_global_stop: - return False, None, None - return self._stoploss_guard(date_now, None) + return False, None, None, None + return self._stoploss_guard(date_now, None, side) - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". :return: Tuple of [bool, until, reason]. If true, this pair will be locked with until """ - return self._stoploss_guard(date_now, pair) + return self._stoploss_guard(date_now, pair, side) From b7cada1edd55e50e3017f55f649bb5ae97f98a76 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 10:29:19 +0200 Subject: [PATCH 03/26] Convert ProtectionReturn to dataclass --- docs/developer.md | 3 ++- freqtrade/plugins/protectionmanager.py | 19 +++++++-------- .../plugins/protections/cooldown_period.py | 17 +++++++++----- freqtrade/plugins/protections/iprotection.py | 16 +++++++++---- .../plugins/protections/low_profit_pairs.py | 20 +++++++++------- .../protections/max_drawdown_protection.py | 23 +++++++++++-------- .../plugins/protections/stoploss_guard.py | 17 +++++++++----- tests/plugins/test_protections.py | 4 ++-- tests/test_freqtradebot.py | 5 ++-- 9 files changed, 74 insertions(+), 50 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 1cc16294b..185bfc92e 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -200,11 +200,12 @@ For that reason, they must implement the following methods: * `global_stop()` * `stop_per_pair()`. -`global_stop()` and `stop_per_pair()` must return a ProtectionReturn tuple, which consists of: +`global_stop()` and `stop_per_pair()` must return a ProtectionReturn object, which consists of: * lock pair - boolean * lock until - datetime - until when should the pair be locked (will be rounded up to the next new candle) * reason - string, used for logging and storage in the database +* lock_side - long, short or '*'. The `until` portion should be calculated using the provided `calculate_lock_end()` method. diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index e8c3fa02d..6a54c4369 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -50,13 +50,10 @@ class ProtectionManager(): result = None for protection_handler in self._protection_handlers: if protection_handler.has_global_stop: - lock, until, reason, lock_side = protection_handler.global_stop( - date_now=now, side=side) - - # Early stopping - first positive result blocks further trades - if lock and until: - if not PairLocks.is_global_lock(until): - result = PairLocks.lock_pair('*', until, reason, now=now) + lock = protection_handler.global_stop(date_now=now, side=side) + if lock and lock.until: + if not PairLocks.is_global_lock(lock.until): + result = PairLocks.lock_pair('*', lock.until, lock.reason, now=now) return result def stop_per_pair( @@ -66,9 +63,9 @@ class ProtectionManager(): result = None for protection_handler in self._protection_handlers: if protection_handler.has_local_stop: - lock, until, reason, lock_side = protection_handler.stop_per_pair( + lock = protection_handler.stop_per_pair( pair=pair, date_now=now, side=side) - if lock and until: - if not PairLocks.is_pair_locked(pair, until): - result = PairLocks.lock_pair(pair, until, reason, now=now) + if lock and lock.until: + if not PairLocks.is_pair_locked(pair, lock.until): + result = PairLocks.lock_pair(pair, lock.until, lock.reason, now=now) return result diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index a75e4fc67..a1d7d4291 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -1,6 +1,7 @@ import logging from datetime import datetime, timedelta +from typing import Optional from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -26,7 +27,7 @@ class CooldownPeriod(IProtection): """ return (f"{self.name} - Cooldown period of {self.stop_duration_str}.") - def _cooldown_period(self, pair: str, date_now: datetime) -> ProtectionReturn: + def _cooldown_period(self, pair: str, date_now: datetime) -> Optional[ProtectionReturn]: """ Get last trade for this pair """ @@ -45,11 +46,15 @@ class CooldownPeriod(IProtection): self.log_once(f"Cooldown for {pair} for {self.stop_duration_str}.", logger.info) until = self.calculate_lock_end([trade], self._stop_duration) - return True, until, self._reason(), None + return ProtectionReturn( + lock=True, + until=until, + reason=self._reason(), + ) - return False, None, None, None + return None - def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -57,9 +62,9 @@ class CooldownPeriod(IProtection): If true, all pairs will be locked with until """ # Not implemented for cooldown period. - return False, None, None, None + return None - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 5f1029eb5..0eff796b3 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -1,8 +1,9 @@ import logging from abc import ABC, abstractmethod +from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import plural @@ -12,8 +13,13 @@ from freqtrade.persistence import LocalTrade logger = logging.getLogger(__name__) -# lock, until, reason, lock_side -ProtectionReturn = Tuple[bool, Optional[datetime], Optional[str], Optional[str]] + +@dataclass +class ProtectionReturn: + lock: bool + until: datetime + reason: Optional[str] + lock_side: Optional[str] = None class IProtection(LoggingMixin, ABC): @@ -81,14 +87,14 @@ class IProtection(LoggingMixin, ABC): """ @abstractmethod - def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". """ @abstractmethod - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 38fd6e734..a4b09bb66 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any, Dict, Optional from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -35,7 +35,7 @@ class LowProfitPairs(IProtection): return (f'{profit} < {self._required_profit} in {self.lookback_period_str}, ' f'locking for {self.stop_duration_str}.') - def _low_profit(self, date_now: datetime, pair: str) -> ProtectionReturn: + def _low_profit(self, date_now: datetime, pair: str) -> Optional[ProtectionReturn]: """ Evaluate recent trades for pair """ @@ -51,7 +51,7 @@ class LowProfitPairs(IProtection): # trades = Trade.get_trades(filters).all() if len(trades) < self._trade_limit: # Not enough trades in the relevant period - return False, None, None, None + return None profit = sum(trade.close_profit for trade in trades if trade.close_profit) if profit < self._required_profit: @@ -60,20 +60,24 @@ class LowProfitPairs(IProtection): f"within {self._lookback_period} minutes.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason(profit), None + return ProtectionReturn( + lock=True, + until=until, + reason=self._reason(profit), + ) - return False, None, None, None + return None - def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". :return: Tuple of [bool, until, reason]. If true, all pairs will be locked with until """ - return False, None, None, None + return None - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index e6cc2ba79..f489522cf 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any, Dict, Optional import pandas as pd @@ -39,7 +39,7 @@ class MaxDrawdown(IProtection): return (f'{drawdown} passed {self._max_allowed_drawdown} in {self.lookback_period_str}, ' f'locking for {self.stop_duration_str}.') - def _max_drawdown(self, date_now: datetime) -> ProtectionReturn: + def _max_drawdown(self, date_now: datetime) -> Optional[ProtectionReturn]: """ Evaluate recent trades for drawdown ... """ @@ -51,14 +51,14 @@ class MaxDrawdown(IProtection): if len(trades) < self._trade_limit: # Not enough trades in the relevant period - return False, None, None, None + return None # Drawdown is always positive try: # TODO: This should use absolute profit calculation, considering account balance. drawdown, _, _, _, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') except ValueError: - return False, None, None, None + return None if drawdown > self._max_allowed_drawdown: self.log_once( @@ -66,11 +66,16 @@ class MaxDrawdown(IProtection): f" within {self.lookback_period_str}.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason(drawdown), None + # return True, until, self._reason(drawdown), None + return ProtectionReturn( + lock=True, + until=until, + reason=self._reason(drawdown), + ) - return False, None, None, None + return None - def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -79,11 +84,11 @@ class MaxDrawdown(IProtection): """ return self._max_drawdown(date_now) - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". :return: Tuple of [bool, until, reason]. If true, this pair will be locked with until """ - return False, None, None, None + return None diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index c8e4dcd21..bb442575e 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -38,7 +38,7 @@ class StoplossGuard(IProtection): f'locking for {self._stop_duration} min.') def _stoploss_guard( - self, date_now: datetime, pair: Optional[str], side: str) -> ProtectionReturn: + self, date_now: datetime, pair: Optional[str], side: str) -> Optional[ProtectionReturn]: """ Evaluate recent trades """ @@ -55,14 +55,19 @@ class StoplossGuard(IProtection): trades = [trade for trade in trades if trade.trade_direction == side] if len(trades) < self._trade_limit: - return False, None, None, None + return None self.log_once(f"Trading stopped due to {self._trade_limit} " f"stoplosses within {self._lookback_period} minutes.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason(), (side if self._only_per_side else None) + return ProtectionReturn( + lock=True, + until=until, + reason=self._reason(), + lock_side=(side if self._only_per_side else None) + ) - def global_stop(self, date_now: datetime, side: str) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -70,10 +75,10 @@ class StoplossGuard(IProtection): If true, all pairs will be locked with until """ if self._disable_global_stop: - return False, None, None, None + return None return self._stoploss_guard(date_now, None, side) - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> ProtectionReturn: + def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 6b69f5481..c8a3b7a82 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -45,9 +45,9 @@ def test_protectionmanager(mocker, default_conf): for handler in freqtrade.protections._protection_handlers: assert handler.name in constants.AVAILABLE_PROTECTIONS if not handler.has_global_stop: - assert handler.global_stop(datetime.utcnow()) == (False, None, None) + assert handler.global_stop(datetime.utcnow(), '*') is None if not handler.has_local_stop: - assert handler.stop_per_pair('XRP/BTC', datetime.utcnow()) == (False, None, None) + assert handler.stop_per_pair('XRP/BTC', datetime.utcnow(), '*') is None @pytest.mark.parametrize('timeframe,expected,protconf', [ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 3737c7c05..0ae36f0fd 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -21,6 +21,7 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock +from freqtrade.plugins.protections.iprotection import ProtectionReturn from freqtrade.worker import Worker from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal, @@ -441,9 +442,9 @@ def test_handle_protections(mocker, default_conf_usdt, fee, is_short): freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade.protections._protection_handlers[1].global_stop = MagicMock( - return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) + return_value=ProtectionReturn(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) create_mock_trades(fee, is_short) - freqtrade.handle_protections('ETC/BTC') + freqtrade.handle_protections('ETC/BTC', '*') send_msg_mock = freqtrade.rpc.send_msg assert send_msg_mock.call_count == 2 assert send_msg_mock.call_args_list[0][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER From 7c79d937e0a82f41015c73666092f403d0a11eb2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 10:58:21 +0200 Subject: [PATCH 04/26] Properly type "side" parameter --- freqtrade/freqtradebot.py | 2 +- freqtrade/optimize/backtesting.py | 3 ++- freqtrade/persistence/models.py | 4 ++-- freqtrade/plugins/protectionmanager.py | 8 +++++--- freqtrade/plugins/protections/cooldown_period.py | 6 ++++-- freqtrade/plugins/protections/iprotection.py | 6 ++++-- freqtrade/plugins/protections/low_profit_pairs.py | 6 ++++-- freqtrade/plugins/protections/max_drawdown_protection.py | 6 ++++-- freqtrade/plugins/protections/stoploss_guard.py | 6 ++++-- 9 files changed, 30 insertions(+), 17 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d3408ada2..833c80735 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1611,7 +1611,7 @@ class FreqtradeBot(LoggingMixin): return False - def handle_protections(self, pair: str, side: str) -> None: + def handle_protections(self, pair: str, side: LongShort) -> None: prot_trig = self.protections.stop_per_pair(pair, side=side) if prot_trig: msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 86c52e737..3c41967e3 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -849,7 +849,8 @@ class Backtesting: return 'short' return None - def run_protections(self, enable_protections, pair: str, current_time: datetime, side: str): + def run_protections( + self, enable_protections, pair: str, current_time: datetime, side: LongShort): if enable_protections: self.protections.stop_per_pair(pair, current_time, side) self.protections.global_stop(current_time, side) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 4aa1c6a4d..98aeacee9 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -13,7 +13,7 @@ from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session from sqlalchemy.pool import StaticPool from sqlalchemy.sql.schema import UniqueConstraint -from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES +from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, LongShort from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.leverage import interest @@ -393,7 +393,7 @@ class LocalTrade(): return "sell" @property - def trade_direction(self) -> str: + def trade_direction(self) -> LongShort: if self.is_short: return "short" else: diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 6a54c4369..d46826605 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -5,6 +5,7 @@ import logging from datetime import datetime, timezone from typing import Dict, List, Optional +from freqtrade.constants import LongShort from freqtrade.persistence import PairLocks from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections import IProtection @@ -44,7 +45,8 @@ class ProtectionManager(): """ return [{p.name: p.short_desc()} for p in self._protection_handlers] - def global_stop(self, now: Optional[datetime] = None, side: str = 'long') -> Optional[PairLock]: + def global_stop(self, now: Optional[datetime] = None, + side: LongShort = 'long') -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) result = None @@ -56,8 +58,8 @@ class ProtectionManager(): result = PairLocks.lock_pair('*', lock.until, lock.reason, now=now) return result - def stop_per_pair( - self, pair, now: Optional[datetime] = None, side: str = 'long') -> Optional[PairLock]: + def stop_per_pair(self, pair, now: Optional[datetime] = None, + side: LongShort = 'long') -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) result = None diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index a1d7d4291..426b8f1b6 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -3,6 +3,7 @@ import logging from datetime import datetime, timedelta from typing import Optional +from freqtrade.constants import LongShort from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -54,7 +55,7 @@ class CooldownPeriod(IProtection): return None - def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -64,7 +65,8 @@ class CooldownPeriod(IProtection): # Not implemented for cooldown period. return None - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 0eff796b3..5ec1c0779 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional +from freqtrade.constants import LongShort from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import plural from freqtrade.mixins import LoggingMixin @@ -87,14 +88,15 @@ class IProtection(LoggingMixin, ABC): """ @abstractmethod - def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". """ @abstractmethod - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index a4b09bb66..7d5d6054d 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -3,6 +3,7 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict, Optional +from freqtrade.constants import LongShort from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -68,7 +69,7 @@ class LowProfitPairs(IProtection): return None - def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -77,7 +78,8 @@ class LowProfitPairs(IProtection): """ return None - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index f489522cf..d759a23dd 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -5,6 +5,7 @@ from typing import Any, Dict, Optional import pandas as pd +from freqtrade.constants import LongShort from freqtrade.data.btanalysis import calculate_max_drawdown from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -75,7 +76,7 @@ class MaxDrawdown(IProtection): return None - def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -84,7 +85,8 @@ class MaxDrawdown(IProtection): """ return self._max_drawdown(date_now) - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index bb442575e..d0ac2783d 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -3,6 +3,7 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict, Optional +from freqtrade.constants import LongShort from freqtrade.enums import ExitType from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -67,7 +68,7 @@ class StoplossGuard(IProtection): lock_side=(side if self._only_per_side else None) ) - def global_stop(self, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -78,7 +79,8 @@ class StoplossGuard(IProtection): return None return self._stoploss_guard(date_now, None, side) - def stop_per_pair(self, pair: str, date_now: datetime, side: str) -> Optional[ProtectionReturn]: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". From 420836b1b20a24ec07b345909871a9743c7bfa36 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 11:23:04 +0200 Subject: [PATCH 05/26] Update test naming --- tests/plugins/test_protections.py | 64 ++++++++++++++++--------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index c8a3b7a82..8ad712e34 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -11,9 +11,10 @@ from tests.conftest import get_patched_freqtradebot, log_has_re def generate_mock_trade(pair: str, fee: float, is_open: bool, - sell_reason: str = ExitType.EXIT_SIGNAL, + exit_reason: str = ExitType.EXIT_SIGNAL, min_ago_open: int = None, min_ago_close: int = None, - profit_rate: float = 0.9 + profit_rate: float = 0.9, + is_short: bool = False, ): open_rate = random.random() @@ -28,11 +29,12 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, is_open=is_open, amount=0.01 / open_rate, exchange='binance', + is_short=is_short, ) trade.recalc_open_trade_value() if not is_open: - trade.close(open_rate * profit_rate) - trade.exit_reason = sell_reason + trade.close(open_rate * (2 - profit_rate if is_short else profit_rate)) + trade.exit_reason = exit_reason return trade @@ -76,8 +78,10 @@ def test_protections_init(mocker, default_conf, timeframe, expected, protconf): assert man._protection_handlers[0]._stop_duration == expected[1] +@pytest.mark.parametrize('is_short', [False, True]) @pytest.mark.usefixtures("init_persistence") -def test_stoploss_guard(mocker, default_conf, fee, caplog): +def test_stoploss_guard(mocker, default_conf, fee, caplog, is_short): + # Active for both sides (long and short) default_conf['protections'] = [{ "method": "StoplossGuard", "lookback_period": 60, @@ -91,8 +95,8 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=200, min_ago_close=30, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, is_short=is_short, )) assert not freqtrade.protections.global_stop() @@ -100,13 +104,13 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): caplog.clear() # This trade does not count, as it's closed too long ago Trade.query.session.add(generate_mock_trade( - 'BCH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=250, min_ago_close=100, + 'BCH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=250, min_ago_close=100, is_short=is_short, )) Trade.query.session.add(generate_mock_trade( - 'ETH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=240, min_ago_close=30, + 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=240, min_ago_close=30, is_short=is_short, )) # 3 Trades closed - but the 2nd has been closed too long ago. assert not freqtrade.protections.global_stop() @@ -114,8 +118,8 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'LTC/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=180, min_ago_close=30, + 'LTC/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=180, min_ago_close=30, is_short=is_short, )) assert freqtrade.protections.global_stop() @@ -148,7 +152,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair caplog.clear() Trade.query.session.add(generate_mock_trade( - pair, fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, profit_rate=0.9, )) @@ -158,12 +162,12 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair caplog.clear() # This trade does not count, as it's closed too long ago Trade.query.session.add(generate_mock_trade( - pair, fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=250, min_ago_close=100, profit_rate=0.9, )) # Trade does not count for per pair stop as it's the wrong pair. Trade.query.session.add(generate_mock_trade( - 'ETH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=240, min_ago_close=30, profit_rate=0.9, )) # 3 Trades closed - but the 2nd has been closed too long ago. @@ -178,7 +182,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair # 2nd Trade that counts with correct pair Trade.query.session.add(generate_mock_trade( - pair, fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=180, min_ago_close=30, profit_rate=0.9, )) @@ -203,7 +207,7 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, )) @@ -213,7 +217,7 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() Trade.query.session.add(generate_mock_trade( - 'ETH/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value, + 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=205, min_ago_close=35, )) @@ -242,7 +246,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=800, min_ago_close=450, profit_rate=0.9, )) @@ -253,7 +257,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=200, min_ago_close=120, profit_rate=0.9, )) @@ -265,14 +269,14 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): # Add positive trade Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=20, min_ago_close=10, profit_rate=1.15, )) assert not freqtrade.protections.stop_per_pair('XRP/BTC') assert not PairLocks.is_pair_locked('XRP/BTC') Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=110, min_ago_close=20, profit_rate=0.8, )) @@ -300,15 +304,15 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) Trade.query.session.add(generate_mock_trade( - 'ETH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) Trade.query.session.add(generate_mock_trade( - 'NEO/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'NEO/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) # No losing trade yet ... so max_drawdown will raise exception @@ -316,7 +320,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not freqtrade.protections.stop_per_pair('XRP/BTC') Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=500, min_ago_close=400, profit_rate=0.9, )) # Not locked with one trade @@ -326,7 +330,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1200, min_ago_close=1100, profit_rate=0.5, )) @@ -339,7 +343,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): # Winning trade ... (should not lock, does not change drawdown!) Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=320, min_ago_close=410, profit_rate=1.5, )) assert not freqtrade.protections.global_stop() @@ -349,7 +353,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): # Add additional negative trade, causing a loss of > 15% Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=20, min_ago_close=10, profit_rate=0.8, )) assert not freqtrade.protections.stop_per_pair('XRP/BTC') From fc201bb4ffbda7e74b67f2674e07160f7aa5c14a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 11:23:26 +0200 Subject: [PATCH 06/26] implement pairlock side further --- freqtrade/persistence/models.py | 5 ++++- freqtrade/persistence/pairlock_middleware.py | 21 ++++++++++++------- freqtrade/plugins/protectionmanager.py | 10 +++++---- freqtrade/plugins/protections/iprotection.py | 2 +- .../plugins/protections/stoploss_guard.py | 2 +- freqtrade/strategy/interface.py | 4 ++-- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 98aeacee9..611b084a9 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -1445,7 +1445,7 @@ class PairLock(_DECL_BASE): f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})') @staticmethod - def query_pair_locks(pair: Optional[str], now: datetime) -> Query: + def query_pair_locks(pair: Optional[str], now: datetime, side: str = '*') -> Query: """ Get all currently active locks for this pair :param pair: Pair to check for. Returns all current locks if pair is empty @@ -1456,6 +1456,9 @@ class PairLock(_DECL_BASE): PairLock.active.is_(True), ] if pair: filters.append(PairLock.pair == pair) + if side != '*': + filters.append(PairLock.direction == side) + return PairLock.query.filter( *filters ) diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index afbd9781b..b8a092365 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -31,7 +31,7 @@ class PairLocks(): @staticmethod def lock_pair(pair: str, until: datetime, reason: str = None, *, - now: datetime = None) -> PairLock: + now: datetime = None, side: str) -> PairLock: """ Create PairLock from now to "until". Uses database by default, unless PairLocks.use_db is set to False, @@ -40,12 +40,14 @@ class PairLocks(): :param until: End time of the lock. Will be rounded up to the next candle. :param reason: Reason string that will be shown as reason for the lock :param now: Current timestamp. Used to determine lock start time. + :param side: Side to lock pair, can be 'long', 'short' or '*' """ lock = PairLock( pair=pair, lock_time=now or datetime.now(timezone.utc), lock_end_time=timeframe_to_next_date(PairLocks.timeframe, until), reason=reason, + direction=side, active=True ) if PairLocks.use_db: @@ -56,7 +58,8 @@ class PairLocks(): return lock @staticmethod - def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: + def get_pair_locks( + pair: Optional[str], now: Optional[datetime] = None, side: str = '*') -> List[PairLock]: """ Get all currently active locks for this pair :param pair: Pair to check for. Returns all current locks if pair is empty @@ -67,12 +70,13 @@ class PairLocks(): now = datetime.now(timezone.utc) if PairLocks.use_db: - return PairLock.query_pair_locks(pair, now).all() + return PairLock.query_pair_locks(pair, now, side).all() else: locks = [lock for lock in PairLocks.locks if ( lock.lock_end_time >= now and lock.active is True and (pair is None or lock.pair == pair) + and (side == '*' or lock.direction == side) )] return locks @@ -134,7 +138,7 @@ class PairLocks(): lock.active = False @staticmethod - def is_global_lock(now: Optional[datetime] = None) -> bool: + def is_global_lock(now: Optional[datetime] = None, side: str = '*') -> bool: """ :param now: Datetime object (generated via datetime.now(timezone.utc)). defaults to datetime.now(timezone.utc) @@ -142,10 +146,10 @@ class PairLocks(): if not now: now = datetime.now(timezone.utc) - return len(PairLocks.get_pair_locks('*', now)) > 0 + return len(PairLocks.get_pair_locks('*', now, side)) > 0 @staticmethod - def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool: + def is_pair_locked(pair: str, now: Optional[datetime] = None, side: str = '*') -> bool: """ :param pair: Pair to check for :param now: Datetime object (generated via datetime.now(timezone.utc)). @@ -154,7 +158,10 @@ class PairLocks(): if not now: now = datetime.now(timezone.utc) - return len(PairLocks.get_pair_locks(pair, now)) > 0 or PairLocks.is_global_lock(now) + return ( + len(PairLocks.get_pair_locks(pair, now, side)) > 0 + or PairLocks.is_global_lock(now, side) + ) @staticmethod def get_all_locks() -> List[PairLock]: diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index d46826605..4868f2c33 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -54,8 +54,9 @@ class ProtectionManager(): if protection_handler.has_global_stop: lock = protection_handler.global_stop(date_now=now, side=side) if lock and lock.until: - if not PairLocks.is_global_lock(lock.until): - result = PairLocks.lock_pair('*', lock.until, lock.reason, now=now) + if not PairLocks.is_global_lock(lock.until, lock.lock_side): + result = PairLocks.lock_pair( + '*', lock.until, lock.reason, now=now, side=lock.lock_side) return result def stop_per_pair(self, pair, now: Optional[datetime] = None, @@ -68,6 +69,7 @@ class ProtectionManager(): lock = protection_handler.stop_per_pair( pair=pair, date_now=now, side=side) if lock and lock.until: - if not PairLocks.is_pair_locked(pair, lock.until): - result = PairLocks.lock_pair(pair, lock.until, lock.reason, now=now) + if not PairLocks.is_pair_locked(pair, lock.until, lock.lock_side): + result = PairLocks.lock_pair( + pair, lock.until, lock.reason, now=now, side=lock.lock_side) return result diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 5ec1c0779..890988226 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -20,7 +20,7 @@ class ProtectionReturn: lock: bool until: datetime reason: Optional[str] - lock_side: Optional[str] = None + lock_side: str = '*' class IProtection(LoggingMixin, ABC): diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index d0ac2783d..1943513ca 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -65,7 +65,7 @@ class StoplossGuard(IProtection): lock=True, until=until, reason=self._reason(), - lock_side=(side if self._only_per_side else None) + lock_side=(side if self._only_per_side else '*') ) def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index afcc1aa99..0a20de08b 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -541,7 +541,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.__class__.__name__ - def lock_pair(self, pair: str, until: datetime, reason: str = None) -> None: + def lock_pair(self, pair: str, until: datetime, reason: str = None, side: str = '*') -> None: """ Locks pair until a given timestamp happens. Locked pairs are not analyzed, and are prevented from opening new trades. @@ -552,7 +552,7 @@ class IStrategy(ABC, HyperStrategyMixin): Needs to be timezone aware `datetime.now(timezone.utc)` :param reason: Optional string explaining why the pair was locked. """ - PairLocks.lock_pair(pair, until, reason) + PairLocks.lock_pair(pair, until, reason, side=side) def unlock_pair(self, pair: str) -> None: """ From 845f960a4e10ee46b188aab47cabcc931279047b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 11:24:15 +0200 Subject: [PATCH 07/26] realign pairlock naming to side --- freqtrade/persistence/migrations.py | 6 +++--- freqtrade/persistence/models.py | 4 ++-- freqtrade/persistence/pairlock_middleware.py | 6 +++--- tests/test_freqtradebot.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index eff2d69f3..93c70b70d 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -219,16 +219,16 @@ def migrate_pairlocks_table( drop_index_on_table(engine, inspector, pairlock_back_name) - direction = get_column_def(cols, 'direction', "'*'") + side = get_column_def(cols, 'side', "'*'") # let SQLAlchemy create the schema as required decl_base.metadata.create_all(engine) # Copy data back - following the correct schema with engine.begin() as connection: connection.execute(text(f"""insert into pairlocks - (id, pair, direction, reason, lock_time, + (id, pair, side, reason, lock_time, lock_end_time, active) - select id, pair, {direction} direction, reason, lock_time, + select id, pair, {side} side, reason, lock_time, lock_end_time, active from {pairlock_back_name} """)) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 611b084a9..1c219610d 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -1429,7 +1429,7 @@ class PairLock(_DECL_BASE): pair = Column(String(25), nullable=False, index=True) # lock direction - long, short or * (for both) - direction = Column(String(25), nullable=False, default="*") + side = Column(String(25), nullable=False, default="*") reason = Column(String(255), nullable=True) # Time the pair was locked (start time) lock_time = Column(DateTime, nullable=False) @@ -1457,7 +1457,7 @@ class PairLock(_DECL_BASE): if pair: filters.append(PairLock.pair == pair) if side != '*': - filters.append(PairLock.direction == side) + filters.append(PairLock.side == side) return PairLock.query.filter( *filters diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index b8a092365..ade92355c 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -31,7 +31,7 @@ class PairLocks(): @staticmethod def lock_pair(pair: str, until: datetime, reason: str = None, *, - now: datetime = None, side: str) -> PairLock: + now: datetime = None, side: str = '*') -> PairLock: """ Create PairLock from now to "until". Uses database by default, unless PairLocks.use_db is set to False, @@ -47,7 +47,7 @@ class PairLocks(): lock_time=now or datetime.now(timezone.utc), lock_end_time=timeframe_to_next_date(PairLocks.timeframe, until), reason=reason, - direction=side, + side=side, active=True ) if PairLocks.use_db: @@ -76,7 +76,7 @@ class PairLocks(): lock.lock_end_time >= now and lock.active is True and (pair is None or lock.pair == pair) - and (side == '*' or lock.direction == side) + and (side == '*' or lock.side == side) )] return locks diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 0ae36f0fd..7bb728c66 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -421,7 +421,7 @@ def test_enter_positions_global_pairlock(default_conf_usdt, ticker_usdt, limit_b assert not log_has_re(message, caplog) caplog.clear() - PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because') + PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because', side='*') n = freqtrade.enter_positions() assert n == 0 assert log_has_re(message, caplog) From 4942d73693e18926029acc0769890b315033d1ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 11:47:28 +0200 Subject: [PATCH 08/26] update pairlock tests --- freqtrade/persistence/models.py | 11 +++++++---- freqtrade/persistence/pairlock_middleware.py | 2 +- tests/plugins/test_pairlocks.py | 16 +++++++++++++++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 1c219610d..1ff38e001 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -7,7 +7,7 @@ from decimal import Decimal from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, - create_engine, desc, func, inspect) + create_engine, desc, func, inspect, or_) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker from sqlalchemy.pool import StaticPool @@ -1441,8 +1441,9 @@ class PairLock(_DECL_BASE): def __repr__(self): lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT) lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT) - return (f'PairLock(id={self.id}, pair={self.pair}, lock_time={lock_time}, ' - f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})') + return ( + f'PairLock(id={self.id}, pair={self.pair}, side={self.side}, lock_time={lock_time}, ' + f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})') @staticmethod def query_pair_locks(pair: Optional[str], now: datetime, side: str = '*') -> Query: @@ -1457,7 +1458,9 @@ class PairLock(_DECL_BASE): if pair: filters.append(PairLock.pair == pair) if side != '*': - filters.append(PairLock.side == side) + filters.append(or_(PairLock.side == side, PairLock.side == '*')) + else: + filters.append(PairLock.side == '*') return PairLock.query.filter( *filters diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index ade92355c..fc727acf5 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -76,7 +76,7 @@ class PairLocks(): lock.lock_end_time >= now and lock.active is True and (pair is None or lock.pair == pair) - and (side == '*' or lock.side == side) + and (lock.side == '*' or lock.side == side) )] return locks diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index f9e5583ed..0ba9bb746 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -21,8 +21,22 @@ def test_PairLocks(use_db): pair = 'ETH/BTC' assert not PairLocks.is_pair_locked(pair) PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) - # ETH/BTC locked for 4 minutes + # ETH/BTC locked for 4 minutes (on both sides) assert PairLocks.is_pair_locked(pair) + assert PairLocks.is_pair_locked(pair, side='long') + assert PairLocks.is_pair_locked(pair, side='short') + + pair = 'BNB/BTC' + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='long') + assert not PairLocks.is_pair_locked(pair) + assert PairLocks.is_pair_locked(pair, side='long') + assert not PairLocks.is_pair_locked(pair, side='short') + + pair = 'BNB/USDT' + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='short') + assert not PairLocks.is_pair_locked(pair) + assert not PairLocks.is_pair_locked(pair, side='long') + assert PairLocks.is_pair_locked(pair, side='short') # XRP/BTC should not be locked now pair = 'XRP/BTC' From b0a8bf3025c63829bdf6487e04e2cbbaa34f4779 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 11:51:33 +0200 Subject: [PATCH 09/26] Show lock side --- freqtrade/persistence/models.py | 1 + freqtrade/rpc/api_server/api_schemas.py | 1 + 2 files changed, 2 insertions(+) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 1ff38e001..843db4691 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -1476,5 +1476,6 @@ class PairLock(_DECL_BASE): 'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc ).timestamp() * 1000), 'reason': self.reason, + 'side': self.side, 'active': self.active, } diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index a9135cce2..d78ea8b78 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -291,6 +291,7 @@ class LockModel(BaseModel): lock_time: str lock_timestamp: int pair: str + side: str reason: str From 144e4da96e947a9c41cf7de54b9d1aff4e12c353 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 12:07:19 +0200 Subject: [PATCH 10/26] Update stoploss guard tests --- tests/plugins/test_protections.py | 40 ++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 8ad712e34..b2dc99610 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -70,7 +70,7 @@ def test_protectionmanager(mocker, default_conf): ('1h', [60, 540], [{"method": "StoplossGuard", "lookback_period_candles": 1, "stop_duration_candles": 9}]), ]) -def test_protections_init(mocker, default_conf, timeframe, expected, protconf): +def test_protections_init(default_conf, timeframe, expected, protconf): default_conf['timeframe'] = timeframe man = ProtectionManager(default_conf, protconf) assert len(man._protection_handlers) == len(protconf) @@ -134,15 +134,19 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog, is_short): @pytest.mark.parametrize('only_per_pair', [False, True]) +@pytest.mark.parametrize('only_per_side', [False, True]) @pytest.mark.usefixtures("init_persistence") -def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair): +def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair, only_per_side): default_conf['protections'] = [{ "method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2, "stop_duration": 60, - "only_per_pair": only_per_pair + "only_per_pair": only_per_pair, + "only_per_side": only_per_side, }] + check_side = 'long' if only_per_side else '*' + is_short = False freqtrade = get_patched_freqtradebot(mocker, default_conf) message = r"Trading stopped due to .*" pair = 'XRP/BTC' @@ -153,7 +157,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair Trade.query.session.add(generate_mock_trade( pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, - min_ago_open=200, min_ago_close=30, profit_rate=0.9, + min_ago_open=200, min_ago_close=30, profit_rate=0.9, is_short=is_short )) assert not freqtrade.protections.stop_per_pair(pair) @@ -163,12 +167,12 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair # This trade does not count, as it's closed too long ago Trade.query.session.add(generate_mock_trade( pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, - min_ago_open=250, min_ago_close=100, profit_rate=0.9, + min_ago_open=250, min_ago_close=100, profit_rate=0.9, is_short=is_short )) # Trade does not count for per pair stop as it's the wrong pair. Trade.query.session.add(generate_mock_trade( 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, - min_ago_open=240, min_ago_close=30, profit_rate=0.9, + min_ago_open=240, min_ago_close=30, profit_rate=0.9, is_short=is_short )) # 3 Trades closed - but the 2nd has been closed too long ago. assert not freqtrade.protections.stop_per_pair(pair) @@ -180,16 +184,34 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair caplog.clear() + # Trade does not count potentially, as it's in the wrong direction + Trade.query.session.add(generate_mock_trade( + pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=150, min_ago_close=25, profit_rate=0.9, is_short=not is_short + )) + freqtrade.protections.stop_per_pair(pair) + assert freqtrade.protections.global_stop() != only_per_pair + assert PairLocks.is_pair_locked(pair, side=check_side) != (only_per_side and only_per_pair) + assert PairLocks.is_global_lock(side=check_side) != only_per_pair + if only_per_side: + assert not PairLocks.is_pair_locked(pair, side='*') + assert not PairLocks.is_global_lock(side='*') + + caplog.clear() + # 2nd Trade that counts with correct pair Trade.query.session.add(generate_mock_trade( pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, - min_ago_open=180, min_ago_close=30, profit_rate=0.9, + min_ago_open=180, min_ago_close=30, profit_rate=0.9, is_short=is_short )) freqtrade.protections.stop_per_pair(pair) assert freqtrade.protections.global_stop() != only_per_pair - assert PairLocks.is_pair_locked(pair) - assert PairLocks.is_global_lock() != only_per_pair + assert PairLocks.is_pair_locked(pair, side=check_side) + assert PairLocks.is_global_lock(side=check_side) != only_per_pair + if only_per_side: + assert not PairLocks.is_pair_locked(pair, side='*') + assert not PairLocks.is_global_lock(side='*') @pytest.mark.usefixtures("init_persistence") From 737bdfe844e575bdbbc9cd9d2a84291fe2e58300 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 14:10:25 +0200 Subject: [PATCH 11/26] Use "side" parameter when calling Pairlocks --- freqtrade/freqtradebot.py | 25 +++++++++++--------- freqtrade/optimize/backtesting.py | 2 +- freqtrade/persistence/migrations.py | 2 +- freqtrade/persistence/pairlock_middleware.py | 9 +++---- freqtrade/plugins/protectionmanager.py | 2 +- freqtrade/strategy/interface.py | 7 +++--- tests/strategy/test_interface.py | 14 +++++------ tests/test_freqtradebot.py | 7 ++++-- tests/test_persistence.py | 5 +++- 9 files changed, 42 insertions(+), 31 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 833c80735..dadfaa5b9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -399,7 +399,10 @@ class FreqtradeBot(LoggingMixin): logger.info("No currency pair in active pair whitelist, " "but checking to exit open trades.") return trades_created - if PairLocks.is_global_lock(): + if PairLocks.is_global_lock(side='*'): + # This only checks for total locks (both sides). + # per-side locks will be evaluated by `is_pair_locked` within create_trade, + # once the direction for the trade is clear. lock = PairLocks.get_pair_longest_lock('*') if lock: self.log_once(f"Global pairlock active until " @@ -433,16 +436,6 @@ class FreqtradeBot(LoggingMixin): analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe) nowtime = analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None - if self.strategy.is_pair_locked(pair, nowtime): - lock = PairLocks.get_pair_longest_lock(pair, nowtime) - if lock: - self.log_once(f"Pair {pair} is still locked until " - f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} " - f"due to {lock.reason}.", - logger.info) - else: - self.log_once(f"Pair {pair} is still locked.", logger.info) - return False # get_free_open_trades is checked before create_trade is called # but it is still used here to prevent opening too many trades within one iteration @@ -458,6 +451,16 @@ class FreqtradeBot(LoggingMixin): ) if signal: + if self.strategy.is_pair_locked(pair, candle_date=nowtime, side=signal): + lock = PairLocks.get_pair_longest_lock(pair, nowtime, signal) + if lock: + self.log_once(f"Pair {pair} {lock.side} is locked until " + f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} " + f"due to {lock.reason}.", + logger.info) + else: + self.log_once(f"Pair {pair} is currently locked.", logger.info) + return False stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge) bid_check_dom = self.config.get('entry_pricing', {}).get('check_depth_of_market', {}) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3c41967e3..260f8e84f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -965,7 +965,7 @@ class Backtesting: and self.trade_slot_available(max_open_trades, open_trade_count_start) and current_time != end_date and trade_dir is not None - and not PairLocks.is_pair_locked(pair, row[DATE_IDX]) + and not PairLocks.is_pair_locked(pair, row[DATE_IDX], trade_dir) ): trade = self._enter_trade(pair, row, trade_dir) if trade: diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 93c70b70d..03f3c3fb9 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -268,7 +268,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None: decl_base, inspector, engine, table_back_name, cols_trades, order_table_bak_name, cols_orders) - if not has_column(cols_pairlocks, 'direction'): + if not has_column(cols_pairlocks, 'side'): logger.info(f"Running database migration for pairlocks - " f"backup: {pairlock_table_bak_name}") diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index fc727acf5..ec57e91fc 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -81,16 +81,17 @@ class PairLocks(): return locks @staticmethod - def get_pair_longest_lock(pair: str, now: Optional[datetime] = None) -> Optional[PairLock]: + def get_pair_longest_lock( + pair: str, now: Optional[datetime] = None, side: str = '*') -> Optional[PairLock]: """ Get the lock that expires the latest for the pair given. """ - locks = PairLocks.get_pair_locks(pair, now) + locks = PairLocks.get_pair_locks(pair, now, side=side) locks = sorted(locks, key=lambda l: l.lock_end_time, reverse=True) return locks[0] if locks else None @staticmethod - def unlock_pair(pair: str, now: Optional[datetime] = None) -> None: + def unlock_pair(pair: str, now: Optional[datetime] = None, side: str = '*') -> None: """ Release all locks for this pair. :param pair: Pair to unlock @@ -101,7 +102,7 @@ class PairLocks(): now = datetime.now(timezone.utc) logger.info(f"Releasing all locks for {pair}.") - locks = PairLocks.get_pair_locks(pair, now) + locks = PairLocks.get_pair_locks(pair, now, side=side) for lock in locks: lock.active = False if PairLocks.use_db: diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 4868f2c33..d33294fa7 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -54,7 +54,7 @@ class ProtectionManager(): if protection_handler.has_global_stop: lock = protection_handler.global_stop(date_now=now, side=side) if lock and lock.until: - if not PairLocks.is_global_lock(lock.until, lock.lock_side): + if not PairLocks.is_global_lock(lock.until, side=lock.lock_side): result = PairLocks.lock_pair( '*', lock.until, lock.reason, now=now, side=lock.lock_side) return result diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 0a20de08b..7d16fc813 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -572,7 +572,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ PairLocks.unlock_reason(reason, datetime.now(timezone.utc)) - def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool: + def is_pair_locked(self, pair: str, *, candle_date: datetime = None, side: str = '*') -> bool: """ Checks if a pair is currently locked The 2nd, optional parameter ensures that locks are applied until the new candle arrives, @@ -580,15 +580,16 @@ class IStrategy(ABC, HyperStrategyMixin): of 2 seconds for an entry order to happen on an old signal. :param pair: "Pair to check" :param candle_date: Date of the last candle. Optional, defaults to current date + :param side: Side to check, can be long, short or '*' :returns: locking state of the pair in question. """ if not candle_date: # Simple call ... - return PairLocks.is_pair_locked(pair) + return PairLocks.is_pair_locked(pair, side=side) else: lock_time = timeframe_to_next_date(self.timeframe, candle_date) - return PairLocks.is_pair_locked(pair, lock_time) + return PairLocks.is_pair_locked(pair, lock_time, side=side) def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index a86d69135..4dc63755f 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -666,23 +666,23 @@ def test_is_pair_locked(default_conf): assert not strategy.is_pair_locked(pair) # latest candle is from 14:20, lock goes to 14:30 - assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-10)) - assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) + assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-10)) + assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-50)) # latest candle is from 14:25 (lock should be lifted) # Since this is the "new candle" available at 14:30 - assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-4)) + assert not strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-4)) # Should not be locked after time expired - assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=10)) + assert not strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=10)) # Change timeframe to 15m strategy.timeframe = '15m' # Candle from 14:14 - lock goes until 14:30 - assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-16)) - assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-15, seconds=-2)) + assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-16)) + assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-15, seconds=-2)) # Candle from 14:15 - lock goes until 14:30 - assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-15)) + assert not strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-15)) def test_is_informative_pairs_callback(default_conf): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 7bb728c66..111638a81 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3796,13 +3796,16 @@ def test_locked_pairs(default_conf_usdt, ticker_usdt, fee, exit_check=ExitCheckTuple(exit_type=ExitType.STOP_LOSS) ) trade.close(ticker_usdt_sell_down()['bid']) - assert freqtrade.strategy.is_pair_locked(trade.pair) + assert freqtrade.strategy.is_pair_locked(trade.pair, side='*') + # Boths sides are locked + assert freqtrade.strategy.is_pair_locked(trade.pair, side='long') + assert freqtrade.strategy.is_pair_locked(trade.pair, side='short') # reinit - should buy other pair. caplog.clear() freqtrade.enter_positions() - assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) + assert log_has_re(fr"Pair {trade.pair} \* is locked.*", caplog) @pytest.mark.parametrize("is_short", [False, True]) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 58d3a4de4..b66c12086 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1471,7 +1471,10 @@ def test_migrate_pairlocks(mocker, default_conf, fee, caplog): assert len(PairLock.query.all()) == 2 assert len(PairLock.query.filter(PairLock.pair == '*').all()) == 1 - assert len(PairLock.query.filter(PairLock.pair == 'ETH/BTC').all()) == 1 + pairlocks = PairLock.query.filter(PairLock.pair == 'ETH/BTC').all() + assert len(pairlocks) == 1 + pairlocks[0].pair == 'ETH/BTC' + pairlocks[0].side == '*' def test_adjust_stop_loss(fee): From 6623192108ff63f52ae6b4a0e8df67c0bc444f9f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 14:38:23 +0200 Subject: [PATCH 12/26] improve doc wording --- docs/includes/protections.md | 4 ++-- freqtrade/strategy/interface.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index a242a6256..bb4a7eb35 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -48,7 +48,7 @@ If `trade_limit` or more trades resulted in stoploss, trading will stop for `sto This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. -Similarly, this protection will by default look at all trades (long and short). For futures bots, setting `only_per_side` will make the bot only one side, and will then only lock this one side. +Similarly, this protection will by default look at all trades (long and short). For futures bots, setting `only_per_side` will make the bot only consider one side, and will then only lock this one side, allowing for example shorts to continue after a series of long stoplosses. The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. @@ -62,7 +62,7 @@ def protections(self): "trade_limit": 4, "stop_duration_candles": 4, "only_per_pair": False, - "only_per_side": True + "only_per_side": False } ] ``` diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7d16fc813..e37fddbe6 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -551,6 +551,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param until: datetime in UTC until the pair should be blocked from opening new trades. Needs to be timezone aware `datetime.now(timezone.utc)` :param reason: Optional string explaining why the pair was locked. + :param side: Side to check, can be long, short or '*' """ PairLocks.lock_pair(pair, until, reason, side=side) From 4de0fdbfca2bb461b0c442661a7bc0624760494b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Apr 2022 14:43:30 +0200 Subject: [PATCH 13/26] Minor edits found during review --- freqtrade/plugins/protections/max_drawdown_protection.py | 1 - freqtrade/plugins/protections/stoploss_guard.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index d759a23dd..7370b2b43 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -67,7 +67,6 @@ class MaxDrawdown(IProtection): f" within {self.lookback_period_str}.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - # return True, until, self._reason(drawdown), None return ProtectionReturn( lock=True, until=until, diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 1943513ca..f9fe039d6 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -51,7 +51,7 @@ class StoplossGuard(IProtection): ExitType.STOPLOSS_ON_EXCHANGE.value) and trade.close_profit and trade.close_profit < 0)] - if self._only_per_side and side: + if self._only_per_side: # Long or short trades only trades = [trade for trade in trades if trade.trade_direction == side] From f96c552c46a56ad5822d764591f7828a5a2dc4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Fri, 29 Apr 2022 22:14:02 +0530 Subject: [PATCH 14/26] Update PULL_REQUEST_TEMPLATE.md Added instructions as comments --- .github/PULL_REQUEST_TEMPLATE.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7c0655b20..25a9761e2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,17 +1,17 @@ -Thank you for sending your pull request. But first, have you included + ## Summary -Explain in one sentence the goal of this PR + Solve the issue: #___ ## Quick changelog -- -- +- +- ## What's new? -*Explain in details what this PR solve or improve. You can include visuals.* + From 2acb68e6e24fdb6920b89fa2992d55d7f7c91bcf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 13:59:23 +0200 Subject: [PATCH 15/26] Move hyperopt-loss functions to their own package --- freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_calmar.py | 0 .../optimize/{ => hyperopt_loss}/hyperopt_loss_max_drawdown.py | 0 .../optimize/{ => hyperopt_loss}/hyperopt_loss_onlyprofit.py | 0 .../{ => hyperopt_loss}/hyperopt_loss_profit_drawdown.py | 0 freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sharpe.py | 0 .../optimize/{ => hyperopt_loss}/hyperopt_loss_sharpe_daily.py | 0 .../{ => hyperopt_loss}/hyperopt_loss_short_trade_dur.py | 0 freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sortino.py | 0 .../optimize/{ => hyperopt_loss}/hyperopt_loss_sortino_daily.py | 0 freqtrade/resolvers/hyperopt_resolver.py | 2 +- tests/optimize/test_hyperoptloss.py | 2 +- 11 files changed, 2 insertions(+), 2 deletions(-) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_calmar.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_max_drawdown.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_onlyprofit.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_profit_drawdown.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sharpe.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sharpe_daily.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_short_trade_dur.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sortino.py (100%) rename freqtrade/optimize/{ => hyperopt_loss}/hyperopt_loss_sortino_daily.py (100%) diff --git a/freqtrade/optimize/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_calmar.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_max_drawdown.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py diff --git a/freqtrade/optimize/hyperopt_loss_onlyprofit.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_onlyprofit.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_onlyprofit.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_onlyprofit.py diff --git a/freqtrade/optimize/hyperopt_loss_profit_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_profit_drawdown.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py diff --git a/freqtrade/optimize/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sharpe.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py diff --git a/freqtrade/optimize/hyperopt_loss_sharpe_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sharpe_daily.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py diff --git a/freqtrade/optimize/hyperopt_loss_short_trade_dur.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_short_trade_dur.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_short_trade_dur.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_short_trade_dur.py diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sortino.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sortino_daily.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index e3c234f60..bcfe5e1d8 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -23,7 +23,7 @@ class HyperOptLossResolver(IResolver): object_type = IHyperOptLoss object_type_str = "HyperoptLoss" user_subdir = USERPATH_HYPEROPTS - initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() + initial_search_path = Path(__file__).parent.parent.joinpath('optimize/hyperopt_loss').resolve() @staticmethod def load_hyperoptloss(config: Dict) -> IHyperOptLoss: diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index e3f6daf6c..4ec80ef49 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import pytest from freqtrade.exceptions import OperationalException -from freqtrade.optimize.hyperopt_loss_short_trade_dur import ShortTradeDurHyperOptLoss +from freqtrade.optimize.hyperopt_loss.hyperopt_loss_short_trade_dur import ShortTradeDurHyperOptLoss from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver From c6c569b77288519c5a7d2016d6a291ee8d68b9ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 14:47:27 +0200 Subject: [PATCH 16/26] chore: split BTAnalyais to metrics --- freqtrade/data/btanalysis.py | 167 +---------------- freqtrade/data/metrics.py | 173 ++++++++++++++++++ .../hyperopt_loss/hyperopt_loss_calmar.py | 2 +- .../hyperopt_loss_max_drawdown.py | 2 +- .../hyperopt_loss_profit_drawdown.py | 2 +- freqtrade/optimize/optimize_reports.py | 4 +- freqtrade/plot/plotting.py | 7 +- .../protections/max_drawdown_protection.py | 2 +- tests/data/test_btanalysis.py | 8 +- tests/test_plotting.py | 3 +- 10 files changed, 190 insertions(+), 180 deletions(-) create mode 100644 freqtrade/data/metrics.py diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0c8e721c0..e29d9ebe4 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -5,7 +5,7 @@ import logging from copy import copy from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import numpy as np import pandas as pd @@ -400,168 +400,3 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame, trades = trades.loc[(trades['open_date'] >= trades_start) & (trades['close_date'] <= trades_stop)] return trades - - -def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float: - """ - Calculate market change based on "column". - Calculation is done by taking the first non-null and the last non-null element of each column - and calculating the pctchange as "(last - first) / first". - Then the results per pair are combined as mean. - - :param data: Dict of Dataframes, dict key should be pair. - :param column: Column in the original dataframes to use - :return: - """ - tmp_means = [] - for pair, df in data.items(): - start = df[column].dropna().iloc[0] - end = df[column].dropna().iloc[-1] - tmp_means.append((end - start) / start) - - return float(np.mean(tmp_means)) - - -def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], - column: str = "close") -> pd.DataFrame: - """ - Combine multiple dataframes "column" - :param data: Dict of Dataframes, dict key should be pair. - :param column: Column in the original dataframes to use - :return: DataFrame with the column renamed to the dict key, and a column - named mean, containing the mean of all pairs. - :raise: ValueError if no data is provided. - """ - df_comb = pd.concat([data[pair].set_index('date').rename( - {column: pair}, axis=1)[pair] for pair in data], axis=1) - - df_comb['mean'] = df_comb.mean(axis=1) - - return df_comb - - -def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, - timeframe: str) -> pd.DataFrame: - """ - Adds a column `col_name` with the cumulative profit for the given trades array. - :param df: DataFrame with date index - :param trades: DataFrame containing trades (requires columns close_date and profit_abs) - :param col_name: Column name that will be assigned the results - :param timeframe: Timeframe used during the operations - :return: Returns df with one additional column, col_name, containing the cumulative profit. - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - from freqtrade.exchange import timeframe_to_minutes - timeframe_minutes = timeframe_to_minutes(timeframe) - # Resample to timeframe to make sure trades match candles - _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' - )[['profit_abs']].sum() - df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum() - # Set first value to 0 - df.loc[df.iloc[0].name, col_name] = 0 - # FFill to get continuous - df[col_name] = df[col_name].ffill() - return df - - -def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str - ) -> pd.DataFrame: - max_drawdown_df = pd.DataFrame() - max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() - max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() - max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] - max_drawdown_df['date'] = profit_results.loc[:, date_col] - return max_drawdown_df - - -def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date', - value_col: str = 'profit_ratio' - ): - """ - Calculate max drawdown and the corresponding close dates - :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) - :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') - :param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio') - :return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown, - high and low time and high and low value. - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - profit_results = trades.sort_values(date_col).reset_index(drop=True) - max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) - - return max_drawdown_df - - -def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', - value_col: str = 'profit_abs', starting_balance: float = 0 - ) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]: - """ - Calculate max drawdown and the corresponding close dates - :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) - :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') - :param value_col: Column in DataFrame to use for values (defaults to 'profit_abs') - :param starting_balance: Portfolio starting balance - properly calculate relative drawdown. - :return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown) - with absolute max drawdown, high and low time and high and low value, - and the relative account drawdown - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - profit_results = trades.sort_values(date_col).reset_index(drop=True) - max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) - - idxmin = max_drawdown_df['drawdown'].idxmin() - if idxmin == 0: - raise ValueError("No losing trade, therefore no drawdown.") - high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] - low_date = profit_results.loc[idxmin, date_col] - high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin] - ['high_value'].idxmax(), 'cumulative'] - low_val = max_drawdown_df.loc[idxmin, 'cumulative'] - max_drawdown_rel = 0.0 - if high_val + starting_balance != 0: - max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance) - - return ( - abs(min(max_drawdown_df['drawdown'])), - high_date, - low_date, - high_val, - low_val, - max_drawdown_rel - ) - - -def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]: - """ - Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane - :param trades: DataFrame containing trades (requires columns close_date and profit_percent) - :param starting_balance: Add starting balance to results, to show the wallets high / low points - :return: Tuple (float, float) with cumsum of profit_abs - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - - csum_df = pd.DataFrame() - csum_df['sum'] = trades['profit_abs'].cumsum() - csum_min = csum_df['sum'].min() + starting_balance - csum_max = csum_df['sum'].max() + starting_balance - - return csum_min, csum_max - - -def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float: - """ - Calculate CAGR - :param days_passed: Days passed between start and ending balance - :param starting_balance: Starting balance - :param final_balance: Final balance to calculate CAGR against - :return: CAGR - """ - return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1 diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py new file mode 100644 index 000000000..44d5ce6ec --- /dev/null +++ b/freqtrade/data/metrics.py @@ -0,0 +1,173 @@ +import logging +from typing import Dict, Tuple + +import numpy as np +import pandas as pd + + +logger = logging.getLogger(__name__) + + +def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float: + """ + Calculate market change based on "column". + Calculation is done by taking the first non-null and the last non-null element of each column + and calculating the pctchange as "(last - first) / first". + Then the results per pair are combined as mean. + + :param data: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: + """ + tmp_means = [] + for pair, df in data.items(): + start = df[column].dropna().iloc[0] + end = df[column].dropna().iloc[-1] + tmp_means.append((end - start) / start) + + return float(np.mean(tmp_means)) + + +def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], + column: str = "close") -> pd.DataFrame: + """ + Combine multiple dataframes "column" + :param data: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: DataFrame with the column renamed to the dict key, and a column + named mean, containing the mean of all pairs. + :raise: ValueError if no data is provided. + """ + df_comb = pd.concat([data[pair].set_index('date').rename( + {column: pair}, axis=1)[pair] for pair in data], axis=1) + + df_comb['mean'] = df_comb.mean(axis=1) + + return df_comb + + +def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, + timeframe: str) -> pd.DataFrame: + """ + Adds a column `col_name` with the cumulative profit for the given trades array. + :param df: DataFrame with date index + :param trades: DataFrame containing trades (requires columns close_date and profit_abs) + :param col_name: Column name that will be assigned the results + :param timeframe: Timeframe used during the operations + :return: Returns df with one additional column, col_name, containing the cumulative profit. + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + from freqtrade.exchange import timeframe_to_minutes + timeframe_minutes = timeframe_to_minutes(timeframe) + # Resample to timeframe to make sure trades match candles + _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' + )[['profit_abs']].sum() + df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum() + # Set first value to 0 + df.loc[df.iloc[0].name, col_name] = 0 + # FFill to get continuous + df[col_name] = df[col_name].ffill() + return df + + +def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str + ) -> pd.DataFrame: + max_drawdown_df = pd.DataFrame() + max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() + max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() + max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] + max_drawdown_df['date'] = profit_results.loc[:, date_col] + return max_drawdown_df + + +def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date', + value_col: str = 'profit_ratio' + ): + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') + :param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio') + :return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown, + high and low time and high and low value. + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col).reset_index(drop=True) + max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) + + return max_drawdown_df + + +def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', + value_col: str = 'profit_abs', starting_balance: float = 0 + ) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]: + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') + :param value_col: Column in DataFrame to use for values (defaults to 'profit_abs') + :param starting_balance: Portfolio starting balance - properly calculate relative drawdown. + :return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown) + with absolute max drawdown, high and low time and high and low value, + and the relative account drawdown + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col).reset_index(drop=True) + max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) + + idxmin = max_drawdown_df['drawdown'].idxmin() + if idxmin == 0: + raise ValueError("No losing trade, therefore no drawdown.") + high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] + low_date = profit_results.loc[idxmin, date_col] + high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin] + ['high_value'].idxmax(), 'cumulative'] + low_val = max_drawdown_df.loc[idxmin, 'cumulative'] + max_drawdown_rel = 0.0 + if high_val + starting_balance != 0: + max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance) + + return ( + abs(min(max_drawdown_df['drawdown'])), + high_date, + low_date, + high_val, + low_val, + max_drawdown_rel + ) + + +def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]: + """ + Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane + :param trades: DataFrame containing trades (requires columns close_date and profit_percent) + :param starting_balance: Add starting balance to results, to show the wallets high / low points + :return: Tuple (float, float) with cumsum of profit_abs + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + + csum_df = pd.DataFrame() + csum_df['sum'] = trades['profit_abs'].cumsum() + csum_min = csum_df['sum'].min() + starting_balance + csum_max = csum_df['sum'].max() + starting_balance + + return csum_min, csum_max + + +def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float: + """ + Calculate CAGR + :param days_passed: Days passed between start and ending balance + :param starting_balance: Starting balance + :param final_balance: Final balance to calculate CAGR against + :return: CAGR + """ + return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1 diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py index 846dae9ea..ea6c151e5 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py @@ -10,7 +10,7 @@ from typing import Any, Dict from pandas import DataFrame -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py index ce955d928..a8af704cd 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py @@ -8,7 +8,7 @@ from datetime import datetime from pandas import DataFrame -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py index 5bd12ff52..ed689edba 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py @@ -9,7 +9,7 @@ individual needs. """ from pandas import DataFrame -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 1d58dc339..9c1a276a9 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -9,8 +9,8 @@ from pandas import DataFrame, to_datetime from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT -from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change, - calculate_max_drawdown) +from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change, + calculate_max_drawdown) from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 5337016f3..773577d7b 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -5,12 +5,13 @@ from typing import Any, Dict, List, Optional import pandas as pd from freqtrade.configuration import TimeRange -from freqtrade.data.btanalysis import (analyze_trade_parallelism, calculate_max_drawdown, - calculate_underwater, combine_dataframes_with_mean, - create_cum_profit, extract_trades_of_period, load_trades) +from freqtrade.data.btanalysis import (analyze_trade_parallelism, extract_trades_of_period, + load_trades) from freqtrade.data.converter import trim_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange, load_data +from freqtrade.data.metrics import (calculate_max_drawdown, calculate_underwater, + combine_dataframes_with_mean, create_cum_profit) from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index b6ef92bd5..4111b7ff4 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -5,7 +5,7 @@ from typing import Any, Dict import pandas as pd -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index eaf703b2d..f9f49e280 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -8,14 +8,14 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN -from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_cagr, - calculate_csum, calculate_market_change, - calculate_max_drawdown, calculate_underwater, - combine_dataframes_with_mean, create_cum_profit, +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, extract_trades_of_period, get_latest_backtest_filename, get_latest_hyperopt_file, load_backtest_data, load_backtest_metadata, load_trades, load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history +from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change, + calculate_max_drawdown, calculate_underwater, + combine_dataframes_with_mean, create_cum_profit) from freqtrade.exceptions import OperationalException from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades from tests.conftest_trades import MOCK_TRADE_COUNT diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 97f367608..65df2d84c 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -10,7 +10,8 @@ from plotly.subplots import make_subplots from freqtrade.commands import start_plot_dataframe, start_plot_profit from freqtrade.configuration import TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data +from freqtrade.data.btanalysis import load_backtest_data +from freqtrade.data.metrics import create_cum_profit from freqtrade.exceptions import OperationalException from freqtrade.plot.plotting import (add_areas, add_indicators, add_profit, create_plotconfig, generate_candlestick_graph, generate_plot_filename, From 4580127fa89f27d432177975b10e1b616842d26e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 14:51:57 +0200 Subject: [PATCH 17/26] Small refactor --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 7456c40cf..dd8d82b13 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1038,7 +1038,7 @@ class Backtesting: timerange: TimeRange): self.progress.init_step(BacktestState.ANALYZE, 0) - logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + logger.info(f"Running backtesting for Strategy {strat.get_strategy_name()}") backtest_start_time = datetime.now(timezone.utc) self._set_strategy(strat) From e4df2b0b966961e188664ed92030c9cb6e454f1f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 14:55:52 +0200 Subject: [PATCH 18/26] Revert unwanted changes --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 25a9761e2..90a10d4da 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,8 +9,8 @@ Solve the issue: #___ ## Quick changelog -- -- +- +- ## What's new? From 11d447cd5af48048c35ac74b0ea74503dae2adc1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 15:28:01 +0200 Subject: [PATCH 19/26] Add support for download-data "until" --- freqtrade/data/history/history_utils.py | 28 +++++++++++++++------ freqtrade/exchange/binance.py | 4 ++- freqtrade/exchange/exchange.py | 11 ++++++--- tests/data/test_history.py | 33 +++++++++++++++++++++---- 4 files changed, 58 insertions(+), 18 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 8560fd29e..d4fe6322a 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -139,8 +139,9 @@ def _load_cached_data_for_updating( timeframe: str, timerange: Optional[TimeRange], data_handler: IDataHandler, - candle_type: CandleType -) -> Tuple[DataFrame, Optional[int]]: + candle_type: CandleType, + prepend: bool = False, +) -> Tuple[DataFrame, Optional[int], Optional[int]]: """ Load cached data to download more data. If timerange is passed in, checks whether data from an before the stored data will be @@ -150,9 +151,12 @@ def _load_cached_data_for_updating( Note: Only used by download_pair_history(). """ start = None + end = None if timerange: if timerange.starttype == 'date': start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + if timerange.stoptype == 'date': + end = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) # Intentionally don't pass timerange in - since we need to load the full dataset. data = data_handler.ohlcv_load(pair, timeframe=timeframe, @@ -160,14 +164,18 @@ def _load_cached_data_for_updating( drop_incomplete=True, warn_no_data=False, candle_type=candle_type) if not data.empty: - if start and start < data.iloc[0]['date']: + if not prepend and start and start < data.iloc[0]['date']: # Earlier data than existing data requested, redownload all data = DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) else: - start = data.iloc[-1]['date'] + if prepend: + end = data.iloc[0]['date'] + else: + start = data.iloc[-1]['date'] start_ms = int(start.timestamp() * 1000) if start else None - return data, start_ms + end_ms = int(end.timestamp() * 1000) if end else None + return data, start_ms, end_ms def _download_pair_history(pair: str, *, @@ -208,9 +216,12 @@ def _download_pair_history(pair: str, *, f'candle type: {candle_type} and store in {datadir}.' ) - data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, - data_handler=data_handler, - candle_type=candle_type) + data, since_ms, until_ms = _load_cached_data_for_updating( + pair, timeframe, timerange, + data_handler=data_handler, + candle_type=candle_type, + prepend=False) + # TODO: Prepend should come from a param logger.debug("Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') @@ -225,6 +236,7 @@ def _download_pair_history(pair: str, *, days=-new_pairs_days).int_timestamp * 1000, is_new_pair=data.empty, candle_type=candle_type, + until_ms=until_ms if until_ms else None ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 8c442cd26..69ae5198a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -95,6 +95,7 @@ class Binance(Exchange): async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, candle_type: CandleType, is_new_pair: bool = False, raise_: bool = False, + until_ms: int = None ) -> Tuple[str, str, str, List]: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date @@ -115,7 +116,8 @@ class Binance(Exchange): since_ms=since_ms, is_new_pair=is_new_pair, raise_=raise_, - candle_type=candle_type + candle_type=candle_type, + until_ms=until_ms, ) def funding_fee_cutoff(self, open_date: datetime): diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 82dcacb51..2ed10ee7a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1645,7 +1645,8 @@ class Exchange: def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, candle_type: CandleType, - is_new_pair: bool = False) -> List: + is_new_pair: bool = False, + until_ms: int = None) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1653,13 +1654,14 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from + :param until_ms: Timestamp in milliseconds to get history up to :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: List with candle (OHLCV) data """ pair, _, _, data = self.loop.run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms, is_new_pair=is_new_pair, - candle_type=candle_type)) + since_ms=since_ms, until_ms=until_ms, + is_new_pair=is_new_pair, candle_type=candle_type)) logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data @@ -1680,6 +1682,7 @@ class Exchange: async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, candle_type: CandleType, is_new_pair: bool = False, raise_: bool = False, + until_ms: int = None ) -> Tuple[str, str, str, List]: """ Download historic ohlcv @@ -1695,7 +1698,7 @@ class Exchange: ) input_coroutines = [self._async_get_candle_history( pair, timeframe, candle_type, since) for since in - range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] + range(since_ms, until_ms or (arrow.utcnow().int_timestamp * 1000), one_call)] data: List = [] # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 0585fa0d4..850849da5 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -223,42 +223,65 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: # timeframe starts earlier than the cached data # should fully update data timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT) assert data.empty assert start_ts == test_data[0][0] - 1000 + assert end_ts is None + + # timeframe starts earlier than the cached data - prepending + + timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) + data, start_ts, end_ts = _load_cached_data_for_updating( + 'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT, True) + assert_frame_equal(data, test_data_df.iloc[:-1]) + assert start_ts == test_data[0][0] - 1000 + assert end_ts == test_data[0][0] # timeframe starts in the center of the cached data # should return the cached data w/o the last item timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT) assert_frame_equal(data, test_data_df.iloc[:-1]) assert test_data[-2][0] <= start_ts < test_data[-1][0] + assert end_ts is None # timeframe starts after the cached data # should return the cached data w/o the last item timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 100, 0) - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT) assert_frame_equal(data, test_data_df.iloc[:-1]) assert test_data[-2][0] <= start_ts < test_data[-1][0] + assert end_ts is None # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'NONEXIST/BTC', '1m', timerange, data_handler, CandleType.SPOT) assert data.empty assert start_ts == (now_ts - 10000) * 1000 + assert end_ts is None + + # no datafile exist + # should return timestamp start and end time time + timerange = TimeRange('date', 'date', now_ts - 1000000, now_ts - 100000) + data, start_ts, end_ts = _load_cached_data_for_updating( + 'NONEXIST/BTC', '1m', timerange, data_handler, CandleType.SPOT) + assert data.empty + assert start_ts == (now_ts - 1000000) * 1000 + assert end_ts == (now_ts - 100000) * 1000 # no datafile exist, no timeframe is set # should return an empty array and None - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'NONEXIST/BTC', '1m', None, data_handler, CandleType.SPOT) assert data.empty assert start_ts is None + assert end_ts is None @pytest.mark.parametrize('candle_type,subdir,file_tail', [ From f6a7e6b785ed8f58a4d9d8584ee2e33d3ac2de43 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 17:24:57 +0200 Subject: [PATCH 20/26] Add prepend option to download-data --- docs/data-download.md | 15 ++++++++++++++- freqtrade/commands/arguments.py | 3 ++- freqtrade/commands/cli_options.py | 5 +++++ freqtrade/commands/data_commands.py | 1 + freqtrade/configuration/configuration.py | 2 ++ freqtrade/data/history/history_utils.py | 23 +++++++++++------------ tests/exchange/test_exchange.py | 14 ++++++++++++++ 7 files changed, 49 insertions(+), 14 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 9bfc1e685..c1caa8722 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -30,6 +30,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [--data-format-trades {json,jsongz,hdf5}] [--trading-mode {spot,margin,futures}] + [--prepend] optional arguments: -h, --help show this help message and exit @@ -62,6 +63,7 @@ optional arguments: `jsongz`). --trading-mode {spot,margin,futures} Select Trading mode + --prepend Allow data prepending. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -157,10 +159,21 @@ freqtrade download-data --exchange binance --pairs .*/USDT - To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.) - To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. - To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days). -- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. Eventually set end dates are ignored. +- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. - Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. +#### Download additional data before the current timerange + +Assuming you downloaded all data from 2022 (`--timerange 20220101-`) - but you'd now like to also backtest with earlier data. +You can do so by using the `--prepend` flag, combined with + +``` bash +freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT --prepend --timerange 20210101-20220101 +``` + +!!! Note + Freqtrade will ignore the end-date in this mode if data is available, updating the end-date to the existing data start point. ### Data format diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 62b79da2e..ff1d16590 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -72,7 +72,8 @@ ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive", "timerange", "download_trades", "exchange", "timeframes", - "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode"] + "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode", + "prepend_data"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index df8966e85..58e208652 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -443,6 +443,11 @@ AVAILABLE_CLI_OPTIONS = { default=['1m', '5m'], nargs='+', ), + "prepend_data": Arg( + '--prepend', + help='Allow data prepending.', + action='store_true', + ), "erase": Arg( '--erase', help='Clean all existing data for the selected exchange/pairs/timeframes.', diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index e41512ccc..a2e2a100a 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -85,6 +85,7 @@ def start_download_data(args: Dict[str, Any]) -> None: new_pairs_days=config['new_pairs_days'], erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'], trading_mode=config.get('trading_mode', 'spot'), + prepend=config.get('prepend_data', False) ) except KeyboardInterrupt: diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index dde56c220..80df6fb3f 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -393,6 +393,8 @@ class Configuration: self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') + self._args_to_config(config, argname='prepend_data', + logstring='Prepend detected. Allowing data prepending.') self._args_to_config(config, argname='erase', logstring='Erase detected. Deleting existing data.') diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index d4fe6322a..f1304607b 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -172,7 +172,6 @@ def _load_cached_data_for_updating( end = data.iloc[0]['date'] else: start = data.iloc[-1]['date'] - start_ms = int(start.timestamp() * 1000) if start else None end_ms = int(end.timestamp() * 1000) if end else None return data, start_ms, end_ms @@ -188,6 +187,7 @@ def _download_pair_history(pair: str, *, timerange: Optional[TimeRange] = None, candle_type: CandleType, erase: bool = False, + prepend: bool = False, ) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters @@ -195,8 +195,6 @@ def _download_pair_history(pair: str, *, exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded - Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pair: pair to download :param timeframe: Timeframe (e.g "5m") :param timerange: range of time to download @@ -211,17 +209,17 @@ def _download_pair_history(pair: str, *, if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.') - logger.info( - f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, ' - f'candle type: {candle_type} and store in {datadir}.' - ) - data, since_ms, until_ms = _load_cached_data_for_updating( pair, timeframe, timerange, data_handler=data_handler, candle_type=candle_type, - prepend=False) - # TODO: Prepend should come from a param + prepend=prepend) + + logger.info(f'Download history data for "{pair}" ({process}), timeframe: {timeframe}, ' + f'candle type: {candle_type} and store in {datadir}.' + f'From {format_ms_time(since_ms) if since_ms else "start"} to ' + f'{format_ms_time(until_ms) if until_ms else "now"}' + ) logger.debug("Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') @@ -269,6 +267,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange: Optional[TimeRange] = None, new_pairs_days: int = 30, erase: bool = False, data_format: str = None, + prepend: bool = False, ) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. @@ -292,7 +291,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange=timerange, data_handler=data_handler, timeframe=str(timeframe), new_pairs_days=new_pairs_days, candle_type=candle_type, - erase=erase) + erase=erase, prepend=prepend) if trading_mode == 'futures': # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. @@ -306,7 +305,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange=timerange, data_handler=data_handler, timeframe=str(tf_mark), new_pairs_days=new_pairs_days, candle_type=funding_candle_type, - erase=erase) + erase=erase, prepend=prepend) return pairs_not_available diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 689ffa4ce..31311cc38 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1983,6 +1983,20 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ assert exchange._api_async.fetch_ohlcv.call_count > 200 assert res[0] == ohlcv[0] + exchange._api_async.fetch_ohlcv.reset_mock() + end_ts = 1_500_500_000_000 + start_ts = 1_500_000_000_000 + respair, restf, _, res = await exchange._async_get_historic_ohlcv( + pair, "5m", since_ms=start_ts, candle_type=candle_type, is_new_pair=False, + until_ms=end_ts + ) + # Required candles + candles = (end_ts - start_ts) / 300_000 + exp = candles // exchange.ohlcv_candle_limit('5m') + 1 + + # Depending on the exchange, this should be called between 1 and 6 times. + assert exchange._api_async.fetch_ohlcv.call_count == exp + @pytest.mark.parametrize('candle_type', [CandleType.FUTURES, CandleType.MARK, CandleType.SPOT]) def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None: From e49b3ef051a3fa0b710a6288bf36ae6d51898dba Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 17:35:11 +0200 Subject: [PATCH 21/26] Improve message formatting --- freqtrade/data/history/history_utils.py | 4 ++-- tests/data/test_history.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index f1304607b..af3a39277 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -215,8 +215,8 @@ def _download_pair_history(pair: str, *, candle_type=candle_type, prepend=prepend) - logger.info(f'Download history data for "{pair}" ({process}), timeframe: {timeframe}, ' - f'candle type: {candle_type} and store in {datadir}.' + logger.info(f'({process}) - Download history data for "{pair}", {timeframe}, ' + f'{candle_type} and store in {datadir}.' f'From {format_ms_time(since_ms) if since_ms else "start"} to ' f'{format_ms_time(until_ms) if until_ms else "now"}' ) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 850849da5..82d4a841c 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -149,8 +149,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC', candle_type=candle_type) assert file.is_file() assert log_has_re( - r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m, ' - r'candle type: spot and store in .*', caplog + r'\(0/1\) - Download history data for "MEME/BTC", 1m, ' + r'spot and store in .*', caplog ) From 8b5d454b50046edfd09d2dbf92d226fbf14346f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 17:44:57 +0200 Subject: [PATCH 22/26] Fix subtle bug in trades download --- freqtrade/data/history/history_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index af3a39277..eb36d2042 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -323,8 +323,9 @@ def _download_trades_history(exchange: Exchange, try: until = None - if (timerange and timerange.starttype == 'date'): - since = timerange.startts * 1000 + if timerange: + if timerange.starttype == 'date': + since = timerange.startts * 1000 if timerange.stoptype == 'date': until = timerange.stopts * 1000 else: From 5c1ac3cf9503ca86d7e52f3f5f199fb356a647fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 19:55:13 +0200 Subject: [PATCH 23/26] Fix caching bug with freqUI backtesting --- freqtrade/rpc/api_server/api_backtest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index a902ea984..41712632b 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -84,6 +84,7 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac lastconfig['enable_protections'] = btconfig.get('enable_protections') lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') + ApiServer._bt.strategylist = [strat] ApiServer._bt.results = {} ApiServer._bt.load_prior_backtest() From 0c921e01161a72c318f1e8a4b0049a2f77f08be0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 Apr 2022 20:08:40 +0200 Subject: [PATCH 24/26] Reorder api_backtesting test sequence --- tests/rpc/test_rpc_apiserver.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 43f783a53..ac2f1c3ec 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1483,7 +1483,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): assert not result['running'] assert result['status_msg'] == 'Backtest reset' ftbot.config['export'] = 'trades' - ftbot.config['backtest_cache'] = 'none' + ftbot.config['backtest_cache'] = 'day' ftbot.config['user_data_dir'] = Path(tmpdir) ftbot.config['exportfilename'] = Path(tmpdir) / "backtest_results" ftbot.config['exportfilename'].mkdir() @@ -1556,19 +1556,19 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): ApiServer._bgtask_running = False - mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', - side_effect=DependencyException()) - rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) - assert log_has("Backtesting caused an error: ", caplog) - - ftbot.config['backtest_cache'] = 'day' - # Rerun backtest (should get previous result) rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) assert_response(rc) result = rc.json() assert log_has_re('Reusing result of previous backtest.*', caplog) + data['stake_amount'] = 101 + + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', + side_effect=DependencyException()) + rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) + assert log_has("Backtesting caused an error: ", caplog) + # Delete backtesting to avoid leakage since the backtest-object may stick around. rc = client_delete(client, f"{BASE_URI}/backtest") assert_response(rc) From d5fc923dcbe02daae51006494be03be6d68a874c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 1 May 2022 09:53:34 +0200 Subject: [PATCH 25/26] Properly validate stoploss existence for optimize commands closes #6740 --- freqtrade/configuration/config_setup.py | 2 +- freqtrade/configuration/config_validation.py | 11 +++++++---- freqtrade/constants.py | 4 ++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 02f2d4089..d49bf61f6 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -22,6 +22,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str # Ensure these modes are using Dry-run config['dry_run'] = True - validate_config_consistency(config) + validate_config_consistency(config, preliminary=True) return config diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 6e4a4b0ef..ee846e7e6 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -39,7 +39,7 @@ def _extend_validator(validator_class): FreqtradeValidator = _extend_validator(Draft4Validator) -def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: +def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> Dict[str, Any]: """ Validate the configuration follow the Config Schema :param conf: Config in JSON format @@ -49,7 +49,10 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE): conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED elif conf.get('runmode', RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT): - conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED + if preliminary: + conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED + else: + conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL else: conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED try: @@ -64,7 +67,7 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: ) -def validate_config_consistency(conf: Dict[str, Any]) -> None: +def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False) -> None: """ Validate the configuration consistency. Should be ran after loading both configuration and strategy, @@ -85,7 +88,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: # validate configuration before returning logger.info('Validating configuration ...') - validate_config_schema(conf) + validate_config_schema(conf, preliminary=preliminary) def _validate_unlimited_amount(conf: Dict[str, Any]) -> None: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1a21ec77f..0ceabe917 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -462,6 +462,10 @@ SCHEMA_BACKTEST_REQUIRED = [ 'dataformat_ohlcv', 'dataformat_trades', ] +SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [ + 'stoploss', + 'minimal_roi', +] SCHEMA_MINIMAL_REQUIRED = [ 'exchange', From 2cedbe5704b9d28facfd4976d0848fec8f523aa3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 1 May 2022 14:50:36 +0200 Subject: [PATCH 26/26] Fix documentation mishap --- docs/data-download.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-download.md b/docs/data-download.md index c1caa8722..681fb717d 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -166,7 +166,7 @@ freqtrade download-data --exchange binance --pairs .*/USDT #### Download additional data before the current timerange Assuming you downloaded all data from 2022 (`--timerange 20220101-`) - but you'd now like to also backtest with earlier data. -You can do so by using the `--prepend` flag, combined with +You can do so by using the `--prepend` flag, combined with `--timerange` - specifying an end-date. ``` bash freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT --prepend --timerange 20210101-20220101