From bfc7898654c82bad60abd59a338dc270c6733253 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 7 May 2022 21:56:22 +0300 Subject: [PATCH 01/32] Report profit only on filled entries. --- freqtrade/rpc/rpc.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 12adc34d1..ba0db72ba 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -177,16 +177,20 @@ class RPC: current_rate = NAN else: current_rate = trade.close_rate - current_profit = trade.calc_profit_ratio(current_rate) - current_profit_abs = trade.calc_profit(current_rate) - current_profit_fiat: Optional[float] = None - # Calculate fiat profit - if self._fiat_converter: - current_profit_fiat = self._fiat_converter.convert_amount( - current_profit_abs, - self._freqtrade.config['stake_currency'], - self._freqtrade.config['fiat_display_currency'] - ) + if len(trade.select_filled_orders(trade.entry_side)) > 0: + logger.warning(trade.select_filled_orders(trade.entry_side)) + current_profit = trade.calc_profit_ratio(current_rate) + current_profit_abs = trade.calc_profit(current_rate) + current_profit_fiat: Optional[float] = None + # Calculate fiat profit + if self._fiat_converter: + current_profit_fiat = self._fiat_converter.convert_amount( + current_profit_abs, + self._freqtrade.config['stake_currency'], + self._freqtrade.config['fiat_display_currency'] + ) + else: + current_profit = current_profit_abs = current_profit_fiat = 0.0 # Calculate guaranteed profit (in case of trailing stop) stoploss_entry_dist = trade.calc_profit(trade.stop_loss) @@ -235,8 +239,12 @@ class RPC: trade.pair, side='exit', is_short=trade.is_short, refresh=False) except (PricingError, ExchangeError): current_rate = NAN - trade_profit = trade.calc_profit(current_rate) - profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}' + if len(trade.select_filled_orders(trade.entry_side)) > 0: + trade_profit = trade.calc_profit(current_rate) + profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}' + else: + trade_profit = 0.0 + profit_str = f'{0.0:.2f}' direction_str = ('S' if trade.is_short else 'L') if nonspot else '' if self._fiat_converter: fiat_profit = self._fiat_converter.convert_amount( @@ -244,7 +252,7 @@ class RPC: stake_currency, fiat_display_currency ) - if fiat_profit and not isnan(fiat_profit): + if not isnan(fiat_profit): profit_str += f" ({fiat_profit:.2f})" fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \ else fiat_profit_sum + fiat_profit From d79b90a98fa90cbf56d510275e94d044e25fda0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Sun, 8 May 2022 12:46:58 +0530 Subject: [PATCH 02/32] consistent exchange name --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d7b1dda37..03cd322a6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -831,7 +831,7 @@ class FreqtradeBot(LoggingMixin): 'type': msg_type, 'buy_tag': trade.enter_tag, 'enter_tag': trade.enter_tag, - 'exchange': self.exchange.name.capitalize(), + 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'leverage': trade.leverage if trade.leverage else None, 'direction': 'Short' if trade.is_short else 'Long', @@ -861,7 +861,7 @@ class FreqtradeBot(LoggingMixin): 'type': RPCMessageType.ENTRY_CANCEL, 'buy_tag': trade.enter_tag, 'enter_tag': trade.enter_tag, - 'exchange': self.exchange.name.capitalize(), + 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'leverage': trade.leverage, 'direction': 'Short' if trade.is_short else 'Long', From f43ae0ea439d6aab7ce07daf119c0a7922824539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Sun, 8 May 2022 13:53:07 +0530 Subject: [PATCH 03/32] logged balance details --- freqtrade/wallets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index d93689a0e..9b91430e2 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -300,7 +300,8 @@ class Wallets: if min_stake_amount is not None and min_stake_amount > max_stake_amount: if self._log: - logger.warning("Minimum stake amount > available balance.") + logger.warning("Minimum stake amount > available balance." + f"{min_stake_amount} > {max_stake_amount}") return 0 if min_stake_amount is not None and stake_amount < min_stake_amount: if self._log: From 1436bc1a709d5870cb55e19dd346b2c9f26677df Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 May 2022 15:30:44 +0200 Subject: [PATCH 04/32] Update list-strategies command closes #6795 --- docs/utils.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 5ef5646c3..6c1b26b01 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -119,6 +119,7 @@ This subcommand is useful for finding problems in your environment with loading usage: freqtrade list-strategies [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--strategy-path PATH] [-1] [--no-color] + [--recursive-strategy-search] optional arguments: -h, --help show this help message and exit @@ -126,6 +127,9 @@ optional arguments: -1, --one-column Print output in one column. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. + --recursive-strategy-search + Recursively search for a strategy in the strategies + folder. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -134,9 +138,10 @@ Common arguments: details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH From 4a7515e66acc49bac0ff2b2d809dad9e44bec651 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 May 2022 16:04:06 +0200 Subject: [PATCH 05/32] Add test for 0.0 case --- freqtrade/rpc/rpc.py | 1 - tests/rpc/test_rpc.py | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ba0db72ba..a98e3f96d 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -178,7 +178,6 @@ class RPC: else: current_rate = trade.close_rate if len(trade.select_filled_orders(trade.entry_side)) > 0: - logger.warning(trade.select_filled_orders(trade.entry_side)) current_profit = trade.calc_profit_ratio(current_rate) current_profit_abs = trade.calc_profit(current_rate) current_profit_fiat: Optional[float] = None diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index f4a2f6099..95645c8ba 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -233,9 +233,20 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING with pytest.raises(RPCException, match=r'.*no active trade*'): rpc._rpc_status_table(default_conf['stake_currency'], 'USD') - + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=False) freqtradebot.enter_positions() + result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') + assert "Since" in headers + assert "Pair" in headers + assert 'instantly' == result[0][2] + assert 'ETH/BTC' in result[0][1] + assert '0.00' == result[0][3] + assert isnan(fiat_profit_sum) + + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=True) + freqtradebot.process() + result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert "Since" in headers assert "Pair" in headers @@ -243,8 +254,8 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert 'ETH/BTC' in result[0][1] assert '-0.41%' == result[0][3] assert isnan(fiat_profit_sum) - # Test with fiatconvert + # Test with fiatconvert rpc._fiat_converter = CryptoToFiatConverter() result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert "Since" in headers From 3221726d85c97afd4ce3e01c50f761750c9799e8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 May 2022 17:27:07 +0200 Subject: [PATCH 06/32] Update migration to use boolean value closes #6794 --- freqtrade/persistence/migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 03f3c3fb9..4d29b3d49 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -99,7 +99,7 @@ def migrate_trades_and_orders_table( liquidation_price = get_column_def(cols, 'liquidation_price', get_column_def(cols, 'isolated_liq', 'null')) # sqlite does not support literals for booleans - is_short = get_column_def(cols, 'is_short', '0') + is_short = get_column_def(cols, 'is_short', 'false') # Margin Properties interest_rate = get_column_def(cols, 'interest_rate', '0.0') From af1a5e044987e0a74ec31ca5a0aba00b1c668b39 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 May 2022 17:38:06 +0200 Subject: [PATCH 07/32] Extract base and Pairlock from models file --- freqtrade/persistence/base.py | 7 ++++ freqtrade/persistence/models.py | 69 ++---------------------------- freqtrade/persistence/pairlock.py | 70 +++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 65 deletions(-) create mode 100644 freqtrade/persistence/base.py create mode 100644 freqtrade/persistence/pairlock.py diff --git a/freqtrade/persistence/base.py b/freqtrade/persistence/base.py new file mode 100644 index 000000000..fb2d561e1 --- /dev/null +++ b/freqtrade/persistence/base.py @@ -0,0 +1,7 @@ + +from typing import Any + +from sqlalchemy.orm import declarative_base + + +_DECL_BASE: Any = declarative_base() diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index cb07a4c6c..92b754d0c 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -7,9 +7,9 @@ from decimal import Decimal from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, - create_engine, desc, func, inspect, or_) + create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError -from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker +from sqlalchemy.orm import Query, relationship, scoped_session, sessionmaker from sqlalchemy.pool import StaticPool from sqlalchemy.sql.schema import UniqueConstraint @@ -17,13 +17,14 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.leverage import interest +from freqtrade.persistence.base import _DECL_BASE from freqtrade.persistence.migrations import check_migrate +from freqtrade.persistence.pairlock import PairLock logger = logging.getLogger(__name__) -_DECL_BASE: Any = declarative_base() _SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' @@ -1419,65 +1420,3 @@ class Trade(_DECL_BASE, LocalTrade): .group_by(Trade.pair) \ .order_by(desc('profit_sum')).first() return best_pair - - -class PairLock(_DECL_BASE): - """ - Pair Locks database model. - """ - __tablename__ = 'pairlocks' - - id = Column(Integer, primary_key=True) - - pair = Column(String(25), nullable=False, index=True) - # lock direction - long, short or * (for both) - side = Column(String(25), nullable=False, default="*") - reason = Column(String(255), nullable=True) - # Time the pair was locked (start time) - lock_time = Column(DateTime, nullable=False) - # Time until the pair is locked (end time) - lock_end_time = Column(DateTime, nullable=False, index=True) - - active = Column(Boolean, nullable=False, default=True, index=True) - - def __repr__(self): - lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT) - lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT) - return ( - f'PairLock(id={self.id}, pair={self.pair}, side={self.side}, lock_time={lock_time}, ' - f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})') - - @staticmethod - def query_pair_locks(pair: Optional[str], now: datetime, side: str = '*') -> Query: - """ - Get all currently active locks for this pair - :param pair: Pair to check for. Returns all current locks if pair is empty - :param now: Datetime object (generated via datetime.now(timezone.utc)). - """ - filters = [PairLock.lock_end_time > now, - # Only active locks - PairLock.active.is_(True), ] - if pair: - filters.append(PairLock.pair == pair) - if side != '*': - filters.append(or_(PairLock.side == side, PairLock.side == '*')) - else: - filters.append(PairLock.side == '*') - - return PairLock.query.filter( - *filters - ) - - def to_json(self) -> Dict[str, Any]: - return { - 'id': self.id, - 'pair': self.pair, - 'lock_time': self.lock_time.strftime(DATETIME_PRINT_FORMAT), - 'lock_timestamp': int(self.lock_time.replace(tzinfo=timezone.utc).timestamp() * 1000), - 'lock_end_time': self.lock_end_time.strftime(DATETIME_PRINT_FORMAT), - 'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc - ).timestamp() * 1000), - 'reason': self.reason, - 'side': self.side, - 'active': self.active, - } diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py new file mode 100644 index 000000000..926c641b0 --- /dev/null +++ b/freqtrade/persistence/pairlock.py @@ -0,0 +1,70 @@ +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from sqlalchemy import Boolean, Column, DateTime, Integer, String, or_ +from sqlalchemy.orm import Query + +from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.persistence.base import _DECL_BASE + + +class PairLock(_DECL_BASE): + """ + Pair Locks database model. + """ + __tablename__ = 'pairlocks' + + id = Column(Integer, primary_key=True) + + pair = Column(String(25), nullable=False, index=True) + # lock direction - long, short or * (for both) + side = Column(String(25), nullable=False, default="*") + reason = Column(String(255), nullable=True) + # Time the pair was locked (start time) + lock_time = Column(DateTime, nullable=False) + # Time until the pair is locked (end time) + lock_end_time = Column(DateTime, nullable=False, index=True) + + active = Column(Boolean, nullable=False, default=True, index=True) + + def __repr__(self): + lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT) + lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT) + return ( + f'PairLock(id={self.id}, pair={self.pair}, side={self.side}, lock_time={lock_time}, ' + f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})') + + @staticmethod + def query_pair_locks(pair: Optional[str], now: datetime, side: str = '*') -> Query: + """ + Get all currently active locks for this pair + :param pair: Pair to check for. Returns all current locks if pair is empty + :param now: Datetime object (generated via datetime.now(timezone.utc)). + """ + filters = [PairLock.lock_end_time > now, + # Only active locks + PairLock.active.is_(True), ] + if pair: + filters.append(PairLock.pair == pair) + if side != '*': + filters.append(or_(PairLock.side == side, PairLock.side == '*')) + else: + filters.append(PairLock.side == '*') + + return PairLock.query.filter( + *filters + ) + + def to_json(self) -> Dict[str, Any]: + return { + 'id': self.id, + 'pair': self.pair, + 'lock_time': self.lock_time.strftime(DATETIME_PRINT_FORMAT), + 'lock_timestamp': int(self.lock_time.replace(tzinfo=timezone.utc).timestamp() * 1000), + 'lock_end_time': self.lock_end_time.strftime(DATETIME_PRINT_FORMAT), + 'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc + ).timestamp() * 1000), + 'reason': self.reason, + 'side': self.side, + 'active': self.active, + } From b58e811b1486ae62e835cbea3e40cf88128243a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 May 2022 17:45:20 +0200 Subject: [PATCH 08/32] Move trade/order Models to their own class --- freqtrade/persistence/__init__.py | 4 +- freqtrade/persistence/models.py | 1341 +------------------------ freqtrade/persistence/trade_model.py | 1346 ++++++++++++++++++++++++++ freqtrade/strategy/interface.py | 6 +- 4 files changed, 1354 insertions(+), 1343 deletions(-) create mode 100644 freqtrade/persistence/trade_model.py diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index d1fcac0ba..ab6e2f6a5 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,5 +1,5 @@ # flake8: noqa: F401 -from freqtrade.persistence.models import (LocalTrade, Order, Trade, clean_dry_run_db, cleanup_db, - init_db) +from freqtrade.persistence.models import clean_dry_run_db, cleanup_db, init_db from freqtrade.persistence.pairlock_middleware import PairLocks +from freqtrade.persistence.trade_model import LocalTrade, Order, Trade diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 92b754d0c..c31e50892 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -2,24 +2,17 @@ This module contains the class to persist trades into SQLite """ import logging -from datetime import datetime, timedelta, timezone -from decimal import Decimal -from typing import Any, Dict, List, Optional -from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, - create_engine, desc, func, inspect) +from sqlalchemy import create_engine, inspect from sqlalchemy.exc import NoSuchModuleError -from sqlalchemy.orm import Query, relationship, scoped_session, sessionmaker +from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.pool import StaticPool -from sqlalchemy.sql.schema import UniqueConstraint -from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort -from freqtrade.enums import ExitType, TradingMode -from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.leverage import interest +from freqtrade.exceptions import OperationalException from freqtrade.persistence.base import _DECL_BASE from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock +from freqtrade.persistence.trade_model import Order, Trade logger = logging.getLogger(__name__) @@ -94,1329 +87,3 @@ def clean_dry_run_db() -> None: if 'dry_run' in trade.open_order_id: trade.open_order_id = None Trade.commit() - - -class Order(_DECL_BASE): - """ - Order database model - Keeps a record of all orders placed on the exchange - - One to many relationship with Trades: - - One trade can have many orders - - One Order can only be associated with one Trade - - Mirrors CCXT Order structure - """ - __tablename__ = 'orders' - # Uniqueness should be ensured over pair, order_id - # its likely that order_id is unique per Pair on some exchanges. - __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) - - id = Column(Integer, primary_key=True) - ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True) - - trade = relationship("Trade", back_populates="orders") - - # order_side can only be 'buy', 'sell' or 'stoploss' - ft_order_side: str = Column(String(25), nullable=False) - ft_pair: str = Column(String(25), nullable=False) - ft_is_open = Column(Boolean, nullable=False, default=True, index=True) - - order_id: str = Column(String(255), nullable=False, index=True) - status = Column(String(255), nullable=True) - symbol = Column(String(25), nullable=True) - order_type: str = Column(String(50), nullable=True) - side = Column(String(25), nullable=True) - price = Column(Float, nullable=True) - average = Column(Float, nullable=True) - amount = Column(Float, nullable=True) - filled = Column(Float, nullable=True) - remaining = Column(Float, nullable=True) - cost = Column(Float, nullable=True) - order_date = Column(DateTime, nullable=True, default=datetime.utcnow) - order_filled_date = Column(DateTime, nullable=True) - order_update_date = Column(DateTime, nullable=True) - - ft_fee_base = Column(Float, nullable=True) - - @property - def order_date_utc(self) -> datetime: - """ Order-date with UTC timezoneinfo""" - return self.order_date.replace(tzinfo=timezone.utc) - - @property - def safe_price(self) -> float: - return self.average or self.price - - @property - def safe_filled(self) -> float: - return self.filled or self.amount or 0.0 - - @property - def safe_fee_base(self) -> float: - return self.ft_fee_base or 0.0 - - @property - def safe_amount_after_fee(self) -> float: - return self.safe_filled - self.safe_fee_base - - def __repr__(self): - - return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, ' - f'side={self.side}, order_type={self.order_type}, status={self.status})') - - def update_from_ccxt_object(self, order): - """ - Update Order from ccxt response - Only updates if fields are available from ccxt - - """ - if self.order_id != str(order['id']): - raise DependencyException("Order-id's don't match") - - self.status = order.get('status', self.status) - self.symbol = order.get('symbol', self.symbol) - self.order_type = order.get('type', self.order_type) - self.side = order.get('side', self.side) - self.price = order.get('price', self.price) - self.amount = order.get('amount', self.amount) - self.filled = order.get('filled', self.filled) - self.average = order.get('average', self.average) - self.remaining = order.get('remaining', self.remaining) - self.cost = order.get('cost', self.cost) - - if 'timestamp' in order and order['timestamp'] is not None: - self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc) - - self.ft_is_open = True - if self.status in NON_OPEN_EXCHANGE_STATES: - self.ft_is_open = False - if (order.get('filled', 0.0) or 0.0) > 0: - self.order_filled_date = datetime.now(timezone.utc) - self.order_update_date = datetime.now(timezone.utc) - - def to_json(self, entry_side: str) -> Dict[str, Any]: - return { - 'pair': self.ft_pair, - 'order_id': self.order_id, - 'status': self.status, - 'amount': self.amount, - 'average': round(self.average, 8) if self.average else 0, - 'safe_price': self.safe_price, - 'cost': self.cost if self.cost else 0, - 'filled': self.filled, - 'ft_order_side': self.ft_order_side, - 'is_open': self.ft_is_open, - 'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT) - if self.order_date else None, - 'order_timestamp': int(self.order_date.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None, - 'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT) - if self.order_filled_date else None, - 'order_filled_timestamp': int(self.order_filled_date.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None, - 'order_type': self.order_type, - 'price': self.price, - 'ft_is_entry': self.ft_order_side == entry_side, - 'remaining': self.remaining, - } - - def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'): - self.order_filled_date = close_date - self.filled = self.amount - self.status = 'closed' - self.ft_is_open = False - if (self.ft_order_side == trade.entry_side - and len(trade.select_filled_orders(trade.entry_side)) == 1): - trade.open_rate = self.price - trade.recalc_open_trade_value() - - @staticmethod - def update_orders(orders: List['Order'], order: Dict[str, Any]): - """ - Get all non-closed orders - useful when trying to batch-update orders - """ - if not isinstance(order, dict): - logger.warning(f"{order} is not a valid response object.") - return - - filtered_orders = [o for o in orders if o.order_id == order.get('id')] - if filtered_orders: - oobj = filtered_orders[0] - oobj.update_from_ccxt_object(order) - Order.query.session.commit() - else: - logger.warning(f"Did not find order for {order}.") - - @staticmethod - def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order': - """ - Parse an order from a ccxt object and return a new order Object. - """ - o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair) - - o.update_from_ccxt_object(order) - return o - - @staticmethod - def get_open_orders() -> List['Order']: - """ - Retrieve open orders from the database - :return: List of open orders - """ - return Order.query.filter(Order.ft_is_open.is_(True)).all() - - -class LocalTrade(): - """ - Trade database model. - Used in backtesting - must be aligned to Trade model! - - """ - use_db: bool = False - # Trades container for backtesting - trades: List['LocalTrade'] = [] - trades_open: List['LocalTrade'] = [] - total_profit: float = 0 - - id: int = 0 - - orders: List[Order] = [] - - exchange: str = '' - pair: str = '' - base_currency: str = '' - stake_currency: str = '' - is_open: bool = True - fee_open: float = 0.0 - fee_open_cost: Optional[float] = None - fee_open_currency: str = '' - fee_close: float = 0.0 - fee_close_cost: Optional[float] = None - fee_close_currency: str = '' - open_rate: float = 0.0 - open_rate_requested: Optional[float] = None - # open_trade_value - calculated via _calc_open_trade_value - open_trade_value: float = 0.0 - close_rate: Optional[float] = None - close_rate_requested: Optional[float] = None - close_profit: Optional[float] = None - close_profit_abs: Optional[float] = None - stake_amount: float = 0.0 - amount: float = 0.0 - amount_requested: Optional[float] = None - open_date: datetime - close_date: Optional[datetime] = None - open_order_id: Optional[str] = None - # absolute value of the stop loss - stop_loss: float = 0.0 - # percentage value of the stop loss - stop_loss_pct: float = 0.0 - # absolute value of the initial stop loss - initial_stop_loss: float = 0.0 - # percentage value of the initial stop loss - initial_stop_loss_pct: Optional[float] = None - # stoploss order id which is on exchange - stoploss_order_id: Optional[str] = None - # last update time of the stoploss order on exchange - stoploss_last_update: Optional[datetime] = None - # absolute value of the highest reached price - max_rate: float = 0.0 - # Lowest price reached - min_rate: float = 0.0 - exit_reason: str = '' - exit_order_status: str = '' - strategy: str = '' - enter_tag: Optional[str] = None - timeframe: Optional[int] = None - - trading_mode: TradingMode = TradingMode.SPOT - - # Leverage trading properties - liquidation_price: Optional[float] = None - is_short: bool = False - leverage: float = 1.0 - - # Margin trading properties - interest_rate: float = 0.0 - - # Futures properties - funding_fees: Optional[float] = None - - @property - def buy_tag(self) -> Optional[str]: - """ - Compatibility between buy_tag (old) and enter_tag (new) - Consider buy_tag deprecated - """ - return self.enter_tag - - @property - def has_no_leverage(self) -> bool: - """Returns true if this is a non-leverage, non-short trade""" - return ((self.leverage == 1.0 or self.leverage is None) and not self.is_short) - - @property - def borrowed(self) -> float: - """ - The amount of currency borrowed from the exchange for leverage trades - If a long trade, the amount is in base currency - If a short trade, the amount is in the other currency being traded - """ - if self.has_no_leverage: - return 0.0 - elif not self.is_short: - return (self.amount * self.open_rate) * ((self.leverage - 1) / self.leverage) - else: - return self.amount - - @property - def open_date_utc(self): - return self.open_date.replace(tzinfo=timezone.utc) - - @property - def close_date_utc(self): - return self.close_date.replace(tzinfo=timezone.utc) - - @property - def enter_side(self) -> str: - """ DEPRECATED, please use entry_side instead""" - # TODO: Please remove me after 2022.5 - return self.entry_side - - @property - def entry_side(self) -> str: - if self.is_short: - return "sell" - else: - return "buy" - - @property - def exit_side(self) -> BuySell: - if self.is_short: - return "buy" - else: - return "sell" - - @property - def trade_direction(self) -> LongShort: - if self.is_short: - return "short" - else: - return "long" - - @property - def safe_base_currency(self) -> str: - """ - Compatibility layer for asset - which can be empty for old trades. - """ - try: - return self.base_currency or self.pair.split('/')[0] - except IndexError: - return '' - - @property - def safe_quote_currency(self) -> str: - """ - Compatibility layer for asset - which can be empty for old trades. - """ - try: - return self.stake_currency or self.pair.split('/')[1].split(':')[0] - except IndexError: - return '' - - def __init__(self, **kwargs): - for key in kwargs: - setattr(self, key, kwargs[key]) - self.recalc_open_trade_value() - if self.trading_mode == TradingMode.MARGIN and self.interest_rate is None: - raise OperationalException( - f"{self.trading_mode.value} trading requires param interest_rate on trades") - - def __repr__(self): - open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' - - return ( - f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' - f'is_short={self.is_short or False}, leverage={self.leverage or 1.0}, ' - f'open_rate={self.open_rate:.8f}, open_since={open_since})' - ) - - def to_json(self) -> Dict[str, Any]: - filled_orders = self.select_filled_orders() - orders = [order.to_json(self.entry_side) for order in filled_orders] - - return { - 'trade_id': self.id, - 'pair': self.pair, - 'base_currency': self.safe_base_currency, - 'quote_currency': self.safe_quote_currency, - 'is_open': self.is_open, - 'exchange': self.exchange, - 'amount': round(self.amount, 8), - 'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None, - 'stake_amount': round(self.stake_amount, 8), - 'strategy': self.strategy, - 'buy_tag': self.enter_tag, - 'enter_tag': self.enter_tag, - 'timeframe': self.timeframe, - - 'fee_open': self.fee_open, - 'fee_open_cost': self.fee_open_cost, - 'fee_open_currency': self.fee_open_currency, - 'fee_close': self.fee_close, - 'fee_close_cost': self.fee_close_cost, - 'fee_close_currency': self.fee_close_currency, - - 'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT), - 'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000), - 'open_rate': self.open_rate, - 'open_rate_requested': self.open_rate_requested, - 'open_trade_value': round(self.open_trade_value, 8), - - 'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT) - if self.close_date else None), - 'close_timestamp': int(self.close_date.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None, - 'close_rate': self.close_rate, - 'close_rate_requested': self.close_rate_requested, - 'close_profit': self.close_profit, # Deprecated - 'close_profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None, - 'close_profit_abs': self.close_profit_abs, # Deprecated - - 'trade_duration_s': (int((self.close_date_utc - self.open_date_utc).total_seconds()) - if self.close_date else None), - 'trade_duration': (int((self.close_date_utc - self.open_date_utc).total_seconds() // 60) - if self.close_date else None), - - 'profit_ratio': self.close_profit, - 'profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None, - 'profit_abs': self.close_profit_abs, - - 'sell_reason': self.exit_reason, # Deprecated - 'exit_reason': self.exit_reason, - 'exit_order_status': self.exit_order_status, - 'stop_loss_abs': self.stop_loss, - 'stop_loss_ratio': self.stop_loss_pct if self.stop_loss_pct else None, - 'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None, - 'stoploss_order_id': self.stoploss_order_id, - 'stoploss_last_update': (self.stoploss_last_update.strftime(DATETIME_PRINT_FORMAT) - if self.stoploss_last_update else None), - 'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None, - 'initial_stop_loss_abs': self.initial_stop_loss, - 'initial_stop_loss_ratio': (self.initial_stop_loss_pct - if self.initial_stop_loss_pct else None), - 'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100 - if self.initial_stop_loss_pct else None), - 'min_rate': self.min_rate, - 'max_rate': self.max_rate, - - 'leverage': self.leverage, - 'interest_rate': self.interest_rate, - 'liquidation_price': self.liquidation_price, - 'is_short': self.is_short, - 'trading_mode': self.trading_mode, - 'funding_fees': self.funding_fees, - 'open_order_id': self.open_order_id, - 'orders': orders, - } - - @staticmethod - def reset_trades() -> None: - """ - Resets all trades. Only active for backtesting mode. - """ - LocalTrade.trades = [] - LocalTrade.trades_open = [] - LocalTrade.total_profit = 0 - - def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None: - """ - Adjust the max_rate and min_rate. - """ - self.max_rate = max(current_price, self.max_rate or self.open_rate) - self.min_rate = min(current_price_low, self.min_rate or self.open_rate) - - def set_isolated_liq(self, liquidation_price: Optional[float]): - """ - Method you should use to set self.liquidation price. - Assures stop_loss is not passed the liquidation price - """ - if not liquidation_price: - return - self.liquidation_price = liquidation_price - - def _set_stop_loss(self, stop_loss: float, percent: float): - """ - Method you should use to set self.stop_loss. - Assures stop_loss is not passed the liquidation price - """ - if self.liquidation_price is not None: - if self.is_short: - sl = min(stop_loss, self.liquidation_price) - else: - sl = max(stop_loss, self.liquidation_price) - else: - sl = stop_loss - - if not self.stop_loss: - self.initial_stop_loss = sl - self.stop_loss = sl - - self.stop_loss_pct = -1 * abs(percent) - self.stoploss_last_update = datetime.utcnow() - - def adjust_stop_loss(self, current_price: float, stoploss: float, - initial: bool = False) -> None: - """ - This adjusts the stop loss to it's most recently observed setting - :param current_price: Current rate the asset is traded - :param stoploss: Stoploss as factor (sample -0.05 -> -5% below current price). - :param initial: Called to initiate stop_loss. - Skips everything if self.stop_loss is already set. - """ - if initial and not (self.stop_loss is None or self.stop_loss == 0): - # Don't modify if called with initial and nothing to do - return - - leverage = self.leverage or 1.0 - if self.is_short: - new_loss = float(current_price * (1 + abs(stoploss / leverage))) - # If trading with leverage, don't set the stoploss below the liquidation price - if self.liquidation_price: - new_loss = min(self.liquidation_price, new_loss) - else: - new_loss = float(current_price * (1 - abs(stoploss / leverage))) - # If trading with leverage, don't set the stoploss below the liquidation price - if self.liquidation_price: - new_loss = max(self.liquidation_price, new_loss) - - # no stop loss assigned yet - if self.initial_stop_loss_pct is None: - logger.debug(f"{self.pair} - Assigning new stoploss...") - self._set_stop_loss(new_loss, stoploss) - self.initial_stop_loss = new_loss - self.initial_stop_loss_pct = -1 * abs(stoploss) - - # evaluate if the stop loss needs to be updated - else: - - higher_stop = new_loss > self.stop_loss - lower_stop = new_loss < self.stop_loss - - # stop losses only walk up, never down!, - # ? But adding more to a leveraged trade would create a lower liquidation price, - # ? decreasing the minimum stoploss - if (higher_stop and not self.is_short) or (lower_stop and self.is_short): - logger.debug(f"{self.pair} - Adjusting stoploss...") - self._set_stop_loss(new_loss, stoploss) - else: - logger.debug(f"{self.pair} - Keeping current stoploss...") - - logger.debug( - f"{self.pair} - Stoploss adjusted. current_price={current_price:.8f}, " - f"open_rate={self.open_rate:.8f}, max_rate={self.max_rate or self.open_rate:.8f}, " - f"initial_stop_loss={self.initial_stop_loss:.8f}, " - f"stop_loss={self.stop_loss:.8f}. " - f"Trailing stoploss saved us: " - f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.") - - def update_trade(self, order: Order) -> None: - """ - Updates this entity with amount and actual open/close rates. - :param order: order retrieved by exchange.fetch_order() - :return: None - """ - - # Ignore open and cancelled orders - if order.status == 'open' or order.safe_price is None: - return - - logger.info(f'Updating trade (id={self.id}) ...') - - if order.ft_order_side == self.entry_side: - # Update open rate and actual amount - self.open_rate = order.safe_price - self.amount = order.safe_amount_after_fee - if self.is_open: - payment = "SELL" if self.is_short else "BUY" - logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') - self.open_order_id = None - self.recalc_trade_from_orders() - elif order.ft_order_side == self.exit_side: - if self.is_open: - payment = "BUY" if self.is_short else "SELL" - # * On margin shorts, you buy a little bit more than the amount (amount + interest) - logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') - self.close(order.safe_price) - elif order.ft_order_side == 'stoploss': - self.stoploss_order_id = None - self.close_rate_requested = self.stop_loss - self.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value - if self.is_open: - logger.info(f'{order.order_type.upper()} is hit for {self}.') - self.close(order.safe_price) - else: - raise ValueError(f'Unknown order type: {order.order_type}') - Trade.commit() - - def close(self, rate: float, *, show_msg: bool = True) -> None: - """ - Sets close_rate to the given rate, calculates total profit - and marks trade as closed - """ - self.close_rate = rate - self.close_date = self.close_date or datetime.utcnow() - self.close_profit = self.calc_profit_ratio() - self.close_profit_abs = self.calc_profit() - self.is_open = False - self.exit_order_status = 'closed' - self.open_order_id = None - if show_msg: - logger.info( - 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', - self - ) - - def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], - side: str) -> None: - """ - Update Fee parameters. Only acts once per side - """ - if self.entry_side == side and self.fee_open_currency is None: - self.fee_open_cost = fee_cost - self.fee_open_currency = fee_currency - if fee_rate is not None: - self.fee_open = fee_rate - # Assume close-fee will fall into the same fee category and take an educated guess - self.fee_close = fee_rate - elif self.exit_side == side and self.fee_close_currency is None: - self.fee_close_cost = fee_cost - self.fee_close_currency = fee_currency - if fee_rate is not None: - self.fee_close = fee_rate - - def fee_updated(self, side: str) -> bool: - """ - Verify if this side (buy / sell) has already been updated - """ - if self.entry_side == side: - return self.fee_open_currency is not None - elif self.exit_side == side: - return self.fee_close_currency is not None - else: - return False - - def update_order(self, order: Dict) -> None: - Order.update_orders(self.orders, order) - - def get_exit_order_count(self) -> int: - """ - Get amount of failed exiting orders - assumes full exits. - """ - return len([o for o in self.orders if o.ft_order_side == self.exit_side]) - - def _calc_open_trade_value(self) -> float: - """ - Calculate the open_rate including open_fee. - :return: Price in of the open trade incl. Fees - """ - open_trade = Decimal(self.amount) * Decimal(self.open_rate) - fees = open_trade * Decimal(self.fee_open) - if self.is_short: - return float(open_trade - fees) - else: - return float(open_trade + fees) - - def recalc_open_trade_value(self) -> None: - """ - Recalculate open_trade_value. - Must be called whenever open_rate, fee_open or is_short is changed. - """ - self.open_trade_value = self._calc_open_trade_value() - - def calculate_interest(self, interest_rate: Optional[float] = None) -> Decimal: - """ - :param interest_rate: interest_charge for borrowing this coin(optional). - If interest_rate is not set self.interest_rate will be used - """ - zero = Decimal(0.0) - # If nothing was borrowed - if self.trading_mode != TradingMode.MARGIN or self.has_no_leverage: - return zero - - open_date = self.open_date.replace(tzinfo=None) - now = (self.close_date or datetime.now(timezone.utc)).replace(tzinfo=None) - sec_per_hour = Decimal(3600) - total_seconds = Decimal((now - open_date).total_seconds()) - hours = total_seconds / sec_per_hour or zero - - rate = Decimal(interest_rate or self.interest_rate) - borrowed = Decimal(self.borrowed) - - return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) - - def _calc_base_close(self, amount: Decimal, rate: Optional[float] = None, - fee: Optional[float] = None) -> Decimal: - - close_trade = Decimal(amount) * Decimal(rate or self.close_rate) # type: ignore - fees = close_trade * Decimal(fee or self.fee_close) - - if self.is_short: - return close_trade + fees - else: - return close_trade - fees - - def calc_close_trade_value(self, rate: Optional[float] = None, - fee: Optional[float] = None, - interest_rate: Optional[float] = None) -> float: - """ - Calculate the close_rate including fee - :param fee: fee to use on the close rate (optional). - If rate is not set self.fee will be used - :param rate: rate to compare with (optional). - If rate is not set self.close_rate will be used - :param interest_rate: interest_charge for borrowing this coin (optional). - If interest_rate is not set self.interest_rate will be used - :return: Price in BTC of the open trade - """ - if rate is None and not self.close_rate: - return 0.0 - - amount = Decimal(self.amount) - trading_mode = self.trading_mode or TradingMode.SPOT - - if trading_mode == TradingMode.SPOT: - return float(self._calc_base_close(amount, rate, fee)) - - elif (trading_mode == TradingMode.MARGIN): - - total_interest = self.calculate_interest(interest_rate) - - if self.is_short: - amount = amount + total_interest - return float(self._calc_base_close(amount, rate, fee)) - else: - # Currency already owned for longs, no need to purchase - return float(self._calc_base_close(amount, rate, fee) - total_interest) - - elif (trading_mode == TradingMode.FUTURES): - funding_fees = self.funding_fees or 0.0 - # Positive funding_fees -> Trade has gained from fees. - # Negative funding_fees -> Trade had to pay the fees. - if self.is_short: - return float(self._calc_base_close(amount, rate, fee)) - funding_fees - else: - return float(self._calc_base_close(amount, rate, fee)) + funding_fees - else: - raise OperationalException( - f"{self.trading_mode.value} trading is not yet available using freqtrade") - - def calc_profit(self, rate: Optional[float] = None, - fee: Optional[float] = None, - interest_rate: Optional[float] = None) -> float: - """ - Calculate the absolute profit in stake currency between Close and Open trade - :param fee: fee to use on the close rate (optional). - If fee is not set self.fee will be used - :param rate: close rate to compare with (optional). - If rate is not set self.close_rate will be used - :param interest_rate: interest_charge for borrowing this coin (optional). - If interest_rate is not set self.interest_rate will be used - :return: profit in stake currency as float - """ - close_trade_value = self.calc_close_trade_value( - rate=(rate or self.close_rate), - fee=(fee or self.fee_close), - interest_rate=(interest_rate or self.interest_rate) - ) - - if self.is_short: - profit = self.open_trade_value - close_trade_value - else: - profit = close_trade_value - self.open_trade_value - return float(f"{profit:.8f}") - - def calc_profit_ratio(self, rate: Optional[float] = None, - fee: Optional[float] = None, - interest_rate: Optional[float] = None) -> float: - """ - Calculates the profit as ratio (including fee). - :param rate: rate to compare with (optional). - If rate is not set self.close_rate will be used - :param fee: fee to use on the close rate (optional). - :param interest_rate: interest_charge for borrowing this coin (optional). - If interest_rate is not set self.interest_rate will be used - :return: profit ratio as float - """ - close_trade_value = self.calc_close_trade_value( - rate=(rate or self.close_rate), - fee=(fee or self.fee_close), - interest_rate=(interest_rate or self.interest_rate) - ) - - short_close_zero = (self.is_short and close_trade_value == 0.0) - long_close_zero = (not self.is_short and self.open_trade_value == 0.0) - leverage = self.leverage or 1.0 - - if (short_close_zero or long_close_zero): - return 0.0 - else: - if self.is_short: - profit_ratio = (1 - (close_trade_value / self.open_trade_value)) * leverage - else: - profit_ratio = ((close_trade_value / self.open_trade_value) - 1) * leverage - - return float(f"{profit_ratio:.8f}") - - def recalc_trade_from_orders(self): - # We need at least 2 entry orders for averaging amounts and rates. - # TODO: this condition could probably be removed - if len(self.select_filled_orders(self.entry_side)) < 2: - self.stake_amount = self.amount * self.open_rate / self.leverage - - # Just in case, still recalc open trade value - self.recalc_open_trade_value() - return - - total_amount = 0.0 - total_stake = 0.0 - for o in self.orders: - if (o.ft_is_open or - (o.ft_order_side != self.entry_side) or - (o.status not in NON_OPEN_EXCHANGE_STATES)): - continue - - tmp_amount = o.safe_amount_after_fee - tmp_price = o.average or o.price - if o.filled is not None: - tmp_amount = o.filled - if tmp_amount > 0.0 and tmp_price is not None: - total_amount += tmp_amount - total_stake += tmp_price * tmp_amount - - if total_amount > 0: - # Leverage not updated, as we don't allow changing leverage through DCA at the moment. - self.open_rate = total_stake / total_amount - self.stake_amount = total_stake / (self.leverage or 1.0) - self.amount = total_amount - self.fee_open_cost = self.fee_open * self.stake_amount - self.recalc_open_trade_value() - if self.stop_loss_pct is not None and self.open_rate is not None: - self.adjust_stop_loss(self.open_rate, self.stop_loss_pct) - - def select_order_by_order_id(self, order_id: str) -> Optional[Order]: - """ - Finds order object by Order id. - :param order_id: Exchange order id - """ - for o in self.orders: - if o.order_id == order_id: - return o - return None - - def select_order( - self, order_side: str = None, is_open: Optional[bool] = None) -> Optional[Order]: - """ - Finds latest order for this orderside and status - :param order_side: ft_order_side of the order (either 'buy', 'sell' or 'stoploss') - :param is_open: Only search for open orders? - :return: latest Order object if it exists, else None - """ - orders = self.orders - if order_side: - orders = [o for o in self.orders if o.ft_order_side == order_side] - if is_open is not None: - orders = [o for o in orders if o.ft_is_open == is_open] - if len(orders) > 0: - return orders[-1] - else: - return None - - def select_filled_orders(self, order_side: Optional[str] = None) -> List['Order']: - """ - Finds filled orders for this orderside. - :param order_side: Side of the order (either 'buy', 'sell', or None) - :return: array of Order objects - """ - return [o for o in self.orders if ((o.ft_order_side == order_side) or (order_side is None)) - and o.ft_is_open is False and - (o.filled or 0) > 0 and - o.status in NON_OPEN_EXCHANGE_STATES] - - @property - def nr_of_successful_entries(self) -> int: - """ - Helper function to count the number of entry orders that have been filled. - :return: int count of entry orders that have been filled for this trade. - """ - - return len(self.select_filled_orders(self.entry_side)) - - @property - def nr_of_successful_exits(self) -> int: - """ - Helper function to count the number of exit orders that have been filled. - :return: int count of exit orders that have been filled for this trade. - """ - return len(self.select_filled_orders(self.exit_side)) - - @property - def nr_of_successful_buys(self) -> int: - """ - Helper function to count the number of buy orders that have been filled. - WARNING: Please use nr_of_successful_entries for short support. - :return: int count of buy orders that have been filled for this trade. - """ - - return len(self.select_filled_orders('buy')) - - @property - def nr_of_successful_sells(self) -> int: - """ - Helper function to count the number of sell orders that have been filled. - WARNING: Please use nr_of_successful_exits for short support. - :return: int count of sell orders that have been filled for this trade. - """ - return len(self.select_filled_orders('sell')) - - @property - def sell_reason(self) -> str: - """ DEPRECATED! Please use exit_reason instead.""" - return self.exit_reason - - @staticmethod - def get_trades_proxy(*, pair: str = None, is_open: bool = None, - open_date: datetime = None, close_date: datetime = None, - ) -> List['LocalTrade']: - """ - Helper function to query Trades. - Returns a List of trades, filtered on the parameters given. - In live mode, converts the filter to a database query and returns all rows - In Backtest mode, uses filters on Trade.trades to get the result. - - :return: unsorted List[Trade] - """ - - # Offline mode - without database - if is_open is not None: - if is_open: - sel_trades = LocalTrade.trades_open - else: - sel_trades = LocalTrade.trades - - else: - # Not used during backtesting, but might be used by a strategy - sel_trades = list(LocalTrade.trades + LocalTrade.trades_open) - - if pair: - sel_trades = [trade for trade in sel_trades if trade.pair == pair] - if open_date: - sel_trades = [trade for trade in sel_trades if trade.open_date > open_date] - if close_date: - sel_trades = [trade for trade in sel_trades if trade.close_date - and trade.close_date > close_date] - - return sel_trades - - @staticmethod - def close_bt_trade(trade): - LocalTrade.trades_open.remove(trade) - LocalTrade.trades.append(trade) - LocalTrade.total_profit += trade.close_profit_abs - - @staticmethod - def add_bt_trade(trade): - if trade.is_open: - LocalTrade.trades_open.append(trade) - else: - LocalTrade.trades.append(trade) - - @staticmethod - def get_open_trades() -> List[Any]: - """ - Query trades from persistence layer - """ - return Trade.get_trades_proxy(is_open=True) - - @staticmethod - def stoploss_reinitialization(desired_stoploss): - """ - Adjust initial Stoploss to desired stoploss for all open trades. - """ - for trade in Trade.get_open_trades(): - logger.info("Found open trade: %s", trade) - - # skip case if trailing-stop changed the stoploss already. - if (trade.stop_loss == trade.initial_stop_loss - and trade.initial_stop_loss_pct != desired_stoploss): - # Stoploss value got changed - - logger.info(f"Stoploss for {trade} needs adjustment...") - # Force reset of stoploss - trade.stop_loss = None - trade.initial_stop_loss_pct = None - trade.adjust_stop_loss(trade.open_rate, desired_stoploss) - logger.info(f"New stoploss: {trade.stop_loss}.") - - -class Trade(_DECL_BASE, LocalTrade): - """ - Trade database model. - Also handles updating and querying trades - - Note: Fields must be aligned with LocalTrade class - """ - __tablename__ = 'trades' - - use_db: bool = True - - id = Column(Integer, primary_key=True) - - orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined") - - exchange = Column(String(25), nullable=False) - pair = Column(String(25), nullable=False, index=True) - base_currency = Column(String(25), nullable=True) - stake_currency = Column(String(25), nullable=True) - is_open = Column(Boolean, nullable=False, default=True, index=True) - fee_open = Column(Float, nullable=False, default=0.0) - fee_open_cost = Column(Float, nullable=True) - fee_open_currency = Column(String(25), nullable=True) - fee_close = Column(Float, nullable=False, default=0.0) - fee_close_cost = Column(Float, nullable=True) - fee_close_currency = Column(String(25), nullable=True) - open_rate: float = Column(Float) - open_rate_requested = Column(Float) - # open_trade_value - calculated via _calc_open_trade_value - open_trade_value = Column(Float) - close_rate: Optional[float] = Column(Float) - close_rate_requested = Column(Float) - close_profit = Column(Float) - close_profit_abs = Column(Float) - stake_amount = Column(Float, nullable=False) - amount = Column(Float) - amount_requested = Column(Float) - open_date = Column(DateTime, nullable=False, default=datetime.utcnow) - close_date = Column(DateTime) - open_order_id = Column(String(255)) - # absolute value of the stop loss - stop_loss = Column(Float, nullable=True, default=0.0) - # percentage value of the stop loss - stop_loss_pct = Column(Float, nullable=True) - # absolute value of the initial stop loss - initial_stop_loss = Column(Float, nullable=True, default=0.0) - # percentage value of the initial stop loss - initial_stop_loss_pct = Column(Float, nullable=True) - # stoploss order id which is on exchange - stoploss_order_id = Column(String(255), nullable=True, index=True) - # last update time of the stoploss order on exchange - stoploss_last_update = Column(DateTime, nullable=True) - # absolute value of the highest reached price - max_rate = Column(Float, nullable=True, default=0.0) - # Lowest price reached - min_rate = Column(Float, nullable=True) - exit_reason = Column(String(100), nullable=True) - exit_order_status = Column(String(100), nullable=True) - strategy = Column(String(100), nullable=True) - enter_tag = Column(String(100), nullable=True) - timeframe = Column(Integer, nullable=True) - - trading_mode = Column(Enum(TradingMode), nullable=True) - - # Leverage trading properties - leverage = Column(Float, nullable=True, default=1.0) - is_short = Column(Boolean, nullable=False, default=False) - liquidation_price = Column(Float, nullable=True) - - # Margin Trading Properties - interest_rate = Column(Float, nullable=False, default=0.0) - - # Futures properties - funding_fees = Column(Float, nullable=True, default=None) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.recalc_open_trade_value() - - def delete(self) -> None: - - for order in self.orders: - Order.query.session.delete(order) - - Trade.query.session.delete(self) - Trade.commit() - - @staticmethod - def commit(): - Trade.query.session.commit() - - @staticmethod - def get_trades_proxy(*, pair: str = None, is_open: bool = None, - open_date: datetime = None, close_date: datetime = None, - ) -> List['LocalTrade']: - """ - Helper function to query Trades.j - Returns a List of trades, filtered on the parameters given. - In live mode, converts the filter to a database query and returns all rows - In Backtest mode, uses filters on Trade.trades to get the result. - - :return: unsorted List[Trade] - """ - if Trade.use_db: - trade_filter = [] - if pair: - trade_filter.append(Trade.pair == pair) - if open_date: - trade_filter.append(Trade.open_date > open_date) - if close_date: - trade_filter.append(Trade.close_date > close_date) - if is_open is not None: - trade_filter.append(Trade.is_open.is_(is_open)) - return Trade.get_trades(trade_filter).all() - else: - return LocalTrade.get_trades_proxy( - pair=pair, is_open=is_open, - open_date=open_date, - close_date=close_date - ) - - @staticmethod - def get_trades(trade_filter=None) -> Query: - """ - Helper function to query Trades using filters. - NOTE: Not supported in Backtesting. - :param trade_filter: Optional filter to apply to trades - Can be either a Filter object, or a List of filters - e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])` - e.g. `(trade_filter=Trade.id == trade_id)` - :return: unsorted query object - """ - if not Trade.use_db: - raise NotImplementedError('`Trade.get_trades()` not supported in backtesting mode.') - if trade_filter is not None: - if not isinstance(trade_filter, list): - trade_filter = [trade_filter] - return Trade.query.filter(*trade_filter) - else: - return Trade.query - - @staticmethod - def get_open_order_trades() -> List['Trade']: - """ - Returns all open trades - NOTE: Not supported in Backtesting. - """ - return Trade.get_trades(Trade.open_order_id.isnot(None)).all() - - @staticmethod - def get_open_trades_without_assigned_fees(): - """ - Returns all open trades which don't have open fees set correctly - NOTE: Not supported in Backtesting. - """ - return Trade.get_trades([Trade.fee_open_currency.is_(None), - Trade.orders.any(), - Trade.is_open.is_(True), - ]).all() - - @staticmethod - def get_closed_trades_without_assigned_fees(): - """ - Returns all closed trades which don't have fees set correctly - NOTE: Not supported in Backtesting. - """ - return Trade.get_trades([Trade.fee_close_currency.is_(None), - Trade.orders.any(), - Trade.is_open.is_(False), - ]).all() - - @staticmethod - def get_total_closed_profit() -> float: - """ - Retrieves total realized profit - """ - if Trade.use_db: - total_profit = Trade.query.with_entities( - func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False)).scalar() - else: - total_profit = sum( - t.close_profit_abs for t in LocalTrade.get_trades_proxy(is_open=False)) - return total_profit or 0 - - @staticmethod - def total_open_trades_stakes() -> float: - """ - Calculates total invested amount in open trades - in stake currency - """ - if Trade.use_db: - total_open_stake_amount = Trade.query.with_entities( - func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar() - else: - total_open_stake_amount = sum( - t.stake_amount for t in LocalTrade.get_trades_proxy(is_open=True)) - return total_open_stake_amount or 0 - - @staticmethod - def get_overall_performance(minutes=None) -> List[Dict[str, Any]]: - """ - Returns List of dicts containing all Trades, including profit and trade count - NOTE: Not supported in Backtesting. - """ - filters = [Trade.is_open.is_(False)] - if minutes: - start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes) - filters.append(Trade.close_date >= start_date) - pair_rates = Trade.query.with_entities( - Trade.pair, - func.sum(Trade.close_profit).label('profit_sum'), - func.sum(Trade.close_profit_abs).label('profit_sum_abs'), - func.count(Trade.pair).label('count') - ).filter(*filters)\ - .group_by(Trade.pair) \ - .order_by(desc('profit_sum_abs')) \ - .all() - return [ - { - 'pair': pair, - 'profit_ratio': profit, - 'profit': round(profit * 100, 2), # Compatibility mode - 'profit_pct': round(profit * 100, 2), - 'profit_abs': profit_abs, - 'count': count - } - for pair, profit, profit_abs, count in pair_rates - ] - - @staticmethod - def get_enter_tag_performance(pair: Optional[str]) -> List[Dict[str, Any]]: - """ - Returns List of dicts containing all Trades, based on buy tag performance - Can either be average for all pairs or a specific pair provided - NOTE: Not supported in Backtesting. - """ - - filters = [Trade.is_open.is_(False)] - if(pair is not None): - filters.append(Trade.pair == pair) - - enter_tag_perf = Trade.query.with_entities( - Trade.enter_tag, - func.sum(Trade.close_profit).label('profit_sum'), - func.sum(Trade.close_profit_abs).label('profit_sum_abs'), - func.count(Trade.pair).label('count') - ).filter(*filters)\ - .group_by(Trade.enter_tag) \ - .order_by(desc('profit_sum_abs')) \ - .all() - - return [ - { - 'enter_tag': enter_tag if enter_tag is not None else "Other", - 'profit_ratio': profit, - 'profit_pct': round(profit * 100, 2), - 'profit_abs': profit_abs, - 'count': count - } - for enter_tag, profit, profit_abs, count in enter_tag_perf - ] - - @staticmethod - def get_exit_reason_performance(pair: Optional[str]) -> List[Dict[str, Any]]: - """ - Returns List of dicts containing all Trades, based on exit reason performance - Can either be average for all pairs or a specific pair provided - NOTE: Not supported in Backtesting. - """ - - filters = [Trade.is_open.is_(False)] - if(pair is not None): - filters.append(Trade.pair == pair) - - sell_tag_perf = Trade.query.with_entities( - Trade.exit_reason, - func.sum(Trade.close_profit).label('profit_sum'), - func.sum(Trade.close_profit_abs).label('profit_sum_abs'), - func.count(Trade.pair).label('count') - ).filter(*filters)\ - .group_by(Trade.exit_reason) \ - .order_by(desc('profit_sum_abs')) \ - .all() - - return [ - { - 'exit_reason': exit_reason if exit_reason is not None else "Other", - 'profit_ratio': profit, - 'profit_pct': round(profit * 100, 2), - 'profit_abs': profit_abs, - 'count': count - } - for exit_reason, profit, profit_abs, count in sell_tag_perf - ] - - @staticmethod - def get_mix_tag_performance(pair: Optional[str]) -> List[Dict[str, Any]]: - """ - Returns List of dicts containing all Trades, based on entry_tag + exit_reason performance - Can either be average for all pairs or a specific pair provided - NOTE: Not supported in Backtesting. - """ - - filters = [Trade.is_open.is_(False)] - if(pair is not None): - filters.append(Trade.pair == pair) - - mix_tag_perf = Trade.query.with_entities( - Trade.id, - Trade.enter_tag, - Trade.exit_reason, - func.sum(Trade.close_profit).label('profit_sum'), - func.sum(Trade.close_profit_abs).label('profit_sum_abs'), - func.count(Trade.pair).label('count') - ).filter(*filters)\ - .group_by(Trade.id) \ - .order_by(desc('profit_sum_abs')) \ - .all() - - return_list: List[Dict] = [] - for id, enter_tag, exit_reason, profit, profit_abs, count in mix_tag_perf: - enter_tag = enter_tag if enter_tag is not None else "Other" - exit_reason = exit_reason if exit_reason is not None else "Other" - - if(exit_reason is not None and enter_tag is not None): - mix_tag = enter_tag + " " + exit_reason - i = 0 - if not any(item["mix_tag"] == mix_tag for item in return_list): - return_list.append({'mix_tag': mix_tag, - 'profit': profit, - 'profit_pct': round(profit * 100, 2), - 'profit_abs': profit_abs, - 'count': count}) - else: - while i < len(return_list): - if return_list[i]["mix_tag"] == mix_tag: - return_list[i] = { - 'mix_tag': mix_tag, - 'profit': profit + return_list[i]["profit"], - 'profit_pct': round(profit + return_list[i]["profit"] * 100, 2), - 'profit_abs': profit_abs + return_list[i]["profit_abs"], - 'count': 1 + return_list[i]["count"]} - i += 1 - - return return_list - - @staticmethod - def get_best_pair(start_date: datetime = datetime.fromtimestamp(0)): - """ - Get best pair with closed trade. - NOTE: Not supported in Backtesting. - :returns: Tuple containing (pair, profit_sum) - """ - best_pair = Trade.query.with_entities( - Trade.pair, func.sum(Trade.close_profit).label('profit_sum') - ).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date)) \ - .group_by(Trade.pair) \ - .order_by(desc('profit_sum')).first() - return best_pair diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py new file mode 100644 index 000000000..bb8c03dd2 --- /dev/null +++ b/freqtrade/persistence/trade_model.py @@ -0,0 +1,1346 @@ +""" +This module contains the class to persist trades into SQLite +""" +import logging +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any, Dict, List, Optional + +from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, + UniqueConstraint, desc, func) +from sqlalchemy.orm import Query, relationship + +from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort +from freqtrade.enums import ExitType, TradingMode +from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.leverage import interest +from freqtrade.persistence.base import _DECL_BASE + + +logger = logging.getLogger(__name__) + + +class Order(_DECL_BASE): + """ + Order database model + Keeps a record of all orders placed on the exchange + + One to many relationship with Trades: + - One trade can have many orders + - One Order can only be associated with one Trade + + Mirrors CCXT Order structure + """ + __tablename__ = 'orders' + # Uniqueness should be ensured over pair, order_id + # its likely that order_id is unique per Pair on some exchanges. + __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) + + id = Column(Integer, primary_key=True) + ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True) + + trade = relationship("Trade", back_populates="orders") + + # order_side can only be 'buy', 'sell' or 'stoploss' + ft_order_side: str = Column(String(25), nullable=False) + ft_pair: str = Column(String(25), nullable=False) + ft_is_open = Column(Boolean, nullable=False, default=True, index=True) + + order_id: str = Column(String(255), nullable=False, index=True) + status = Column(String(255), nullable=True) + symbol = Column(String(25), nullable=True) + order_type: str = Column(String(50), nullable=True) + side = Column(String(25), nullable=True) + price = Column(Float, nullable=True) + average = Column(Float, nullable=True) + amount = Column(Float, nullable=True) + filled = Column(Float, nullable=True) + remaining = Column(Float, nullable=True) + cost = Column(Float, nullable=True) + order_date = Column(DateTime, nullable=True, default=datetime.utcnow) + order_filled_date = Column(DateTime, nullable=True) + order_update_date = Column(DateTime, nullable=True) + + ft_fee_base = Column(Float, nullable=True) + + @property + def order_date_utc(self) -> datetime: + """ Order-date with UTC timezoneinfo""" + return self.order_date.replace(tzinfo=timezone.utc) + + @property + def safe_price(self) -> float: + return self.average or self.price + + @property + def safe_filled(self) -> float: + return self.filled or self.amount or 0.0 + + @property + def safe_fee_base(self) -> float: + return self.ft_fee_base or 0.0 + + @property + def safe_amount_after_fee(self) -> float: + return self.safe_filled - self.safe_fee_base + + def __repr__(self): + + return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, ' + f'side={self.side}, order_type={self.order_type}, status={self.status})') + + def update_from_ccxt_object(self, order): + """ + Update Order from ccxt response + Only updates if fields are available from ccxt - + """ + if self.order_id != str(order['id']): + raise DependencyException("Order-id's don't match") + + self.status = order.get('status', self.status) + self.symbol = order.get('symbol', self.symbol) + self.order_type = order.get('type', self.order_type) + self.side = order.get('side', self.side) + self.price = order.get('price', self.price) + self.amount = order.get('amount', self.amount) + self.filled = order.get('filled', self.filled) + self.average = order.get('average', self.average) + self.remaining = order.get('remaining', self.remaining) + self.cost = order.get('cost', self.cost) + + if 'timestamp' in order and order['timestamp'] is not None: + self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc) + + self.ft_is_open = True + if self.status in NON_OPEN_EXCHANGE_STATES: + self.ft_is_open = False + if (order.get('filled', 0.0) or 0.0) > 0: + self.order_filled_date = datetime.now(timezone.utc) + self.order_update_date = datetime.now(timezone.utc) + + def to_json(self, entry_side: str) -> Dict[str, Any]: + return { + 'pair': self.ft_pair, + 'order_id': self.order_id, + 'status': self.status, + 'amount': self.amount, + 'average': round(self.average, 8) if self.average else 0, + 'safe_price': self.safe_price, + 'cost': self.cost if self.cost else 0, + 'filled': self.filled, + 'ft_order_side': self.ft_order_side, + 'is_open': self.ft_is_open, + 'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT) + if self.order_date else None, + 'order_timestamp': int(self.order_date.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None, + 'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT) + if self.order_filled_date else None, + 'order_filled_timestamp': int(self.order_filled_date.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None, + 'order_type': self.order_type, + 'price': self.price, + 'ft_is_entry': self.ft_order_side == entry_side, + 'remaining': self.remaining, + } + + def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'): + self.order_filled_date = close_date + self.filled = self.amount + self.status = 'closed' + self.ft_is_open = False + if (self.ft_order_side == trade.entry_side + and len(trade.select_filled_orders(trade.entry_side)) == 1): + trade.open_rate = self.price + trade.recalc_open_trade_value() + + @staticmethod + def update_orders(orders: List['Order'], order: Dict[str, Any]): + """ + Get all non-closed orders - useful when trying to batch-update orders + """ + if not isinstance(order, dict): + logger.warning(f"{order} is not a valid response object.") + return + + filtered_orders = [o for o in orders if o.order_id == order.get('id')] + if filtered_orders: + oobj = filtered_orders[0] + oobj.update_from_ccxt_object(order) + Order.query.session.commit() + else: + logger.warning(f"Did not find order for {order}.") + + @staticmethod + def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order': + """ + Parse an order from a ccxt object and return a new order Object. + """ + o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair) + + o.update_from_ccxt_object(order) + return o + + @staticmethod + def get_open_orders() -> List['Order']: + """ + Retrieve open orders from the database + :return: List of open orders + """ + return Order.query.filter(Order.ft_is_open.is_(True)).all() + + +class LocalTrade(): + """ + Trade database model. + Used in backtesting - must be aligned to Trade model! + + """ + use_db: bool = False + # Trades container for backtesting + trades: List['LocalTrade'] = [] + trades_open: List['LocalTrade'] = [] + total_profit: float = 0 + + id: int = 0 + + orders: List[Order] = [] + + exchange: str = '' + pair: str = '' + base_currency: str = '' + stake_currency: str = '' + is_open: bool = True + fee_open: float = 0.0 + fee_open_cost: Optional[float] = None + fee_open_currency: str = '' + fee_close: float = 0.0 + fee_close_cost: Optional[float] = None + fee_close_currency: str = '' + open_rate: float = 0.0 + open_rate_requested: Optional[float] = None + # open_trade_value - calculated via _calc_open_trade_value + open_trade_value: float = 0.0 + close_rate: Optional[float] = None + close_rate_requested: Optional[float] = None + close_profit: Optional[float] = None + close_profit_abs: Optional[float] = None + stake_amount: float = 0.0 + amount: float = 0.0 + amount_requested: Optional[float] = None + open_date: datetime + close_date: Optional[datetime] = None + open_order_id: Optional[str] = None + # absolute value of the stop loss + stop_loss: float = 0.0 + # percentage value of the stop loss + stop_loss_pct: float = 0.0 + # absolute value of the initial stop loss + initial_stop_loss: float = 0.0 + # percentage value of the initial stop loss + initial_stop_loss_pct: Optional[float] = None + # stoploss order id which is on exchange + stoploss_order_id: Optional[str] = None + # last update time of the stoploss order on exchange + stoploss_last_update: Optional[datetime] = None + # absolute value of the highest reached price + max_rate: float = 0.0 + # Lowest price reached + min_rate: float = 0.0 + exit_reason: str = '' + exit_order_status: str = '' + strategy: str = '' + enter_tag: Optional[str] = None + timeframe: Optional[int] = None + + trading_mode: TradingMode = TradingMode.SPOT + + # Leverage trading properties + liquidation_price: Optional[float] = None + is_short: bool = False + leverage: float = 1.0 + + # Margin trading properties + interest_rate: float = 0.0 + + # Futures properties + funding_fees: Optional[float] = None + + @property + def buy_tag(self) -> Optional[str]: + """ + Compatibility between buy_tag (old) and enter_tag (new) + Consider buy_tag deprecated + """ + return self.enter_tag + + @property + def has_no_leverage(self) -> bool: + """Returns true if this is a non-leverage, non-short trade""" + return ((self.leverage == 1.0 or self.leverage is None) and not self.is_short) + + @property + def borrowed(self) -> float: + """ + The amount of currency borrowed from the exchange for leverage trades + If a long trade, the amount is in base currency + If a short trade, the amount is in the other currency being traded + """ + if self.has_no_leverage: + return 0.0 + elif not self.is_short: + return (self.amount * self.open_rate) * ((self.leverage - 1) / self.leverage) + else: + return self.amount + + @property + def open_date_utc(self): + return self.open_date.replace(tzinfo=timezone.utc) + + @property + def close_date_utc(self): + return self.close_date.replace(tzinfo=timezone.utc) + + @property + def enter_side(self) -> str: + """ DEPRECATED, please use entry_side instead""" + # TODO: Please remove me after 2022.5 + return self.entry_side + + @property + def entry_side(self) -> str: + if self.is_short: + return "sell" + else: + return "buy" + + @property + def exit_side(self) -> BuySell: + if self.is_short: + return "buy" + else: + return "sell" + + @property + def trade_direction(self) -> LongShort: + if self.is_short: + return "short" + else: + return "long" + + @property + def safe_base_currency(self) -> str: + """ + Compatibility layer for asset - which can be empty for old trades. + """ + try: + return self.base_currency or self.pair.split('/')[0] + except IndexError: + return '' + + @property + def safe_quote_currency(self) -> str: + """ + Compatibility layer for asset - which can be empty for old trades. + """ + try: + return self.stake_currency or self.pair.split('/')[1].split(':')[0] + except IndexError: + return '' + + def __init__(self, **kwargs): + for key in kwargs: + setattr(self, key, kwargs[key]) + self.recalc_open_trade_value() + if self.trading_mode == TradingMode.MARGIN and self.interest_rate is None: + raise OperationalException( + f"{self.trading_mode.value} trading requires param interest_rate on trades") + + def __repr__(self): + open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' + + return ( + f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' + f'is_short={self.is_short or False}, leverage={self.leverage or 1.0}, ' + f'open_rate={self.open_rate:.8f}, open_since={open_since})' + ) + + def to_json(self) -> Dict[str, Any]: + filled_orders = self.select_filled_orders() + orders = [order.to_json(self.entry_side) for order in filled_orders] + + return { + 'trade_id': self.id, + 'pair': self.pair, + 'base_currency': self.safe_base_currency, + 'quote_currency': self.safe_quote_currency, + 'is_open': self.is_open, + 'exchange': self.exchange, + 'amount': round(self.amount, 8), + 'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None, + 'stake_amount': round(self.stake_amount, 8), + 'strategy': self.strategy, + 'buy_tag': self.enter_tag, + 'enter_tag': self.enter_tag, + 'timeframe': self.timeframe, + + 'fee_open': self.fee_open, + 'fee_open_cost': self.fee_open_cost, + 'fee_open_currency': self.fee_open_currency, + 'fee_close': self.fee_close, + 'fee_close_cost': self.fee_close_cost, + 'fee_close_currency': self.fee_close_currency, + + 'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT), + 'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000), + 'open_rate': self.open_rate, + 'open_rate_requested': self.open_rate_requested, + 'open_trade_value': round(self.open_trade_value, 8), + + 'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT) + if self.close_date else None), + 'close_timestamp': int(self.close_date.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None, + 'close_rate': self.close_rate, + 'close_rate_requested': self.close_rate_requested, + 'close_profit': self.close_profit, # Deprecated + 'close_profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None, + 'close_profit_abs': self.close_profit_abs, # Deprecated + + 'trade_duration_s': (int((self.close_date_utc - self.open_date_utc).total_seconds()) + if self.close_date else None), + 'trade_duration': (int((self.close_date_utc - self.open_date_utc).total_seconds() // 60) + if self.close_date else None), + + 'profit_ratio': self.close_profit, + 'profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None, + 'profit_abs': self.close_profit_abs, + + 'sell_reason': self.exit_reason, # Deprecated + 'exit_reason': self.exit_reason, + 'exit_order_status': self.exit_order_status, + 'stop_loss_abs': self.stop_loss, + 'stop_loss_ratio': self.stop_loss_pct if self.stop_loss_pct else None, + 'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None, + 'stoploss_order_id': self.stoploss_order_id, + 'stoploss_last_update': (self.stoploss_last_update.strftime(DATETIME_PRINT_FORMAT) + if self.stoploss_last_update else None), + 'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None, + 'initial_stop_loss_abs': self.initial_stop_loss, + 'initial_stop_loss_ratio': (self.initial_stop_loss_pct + if self.initial_stop_loss_pct else None), + 'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100 + if self.initial_stop_loss_pct else None), + 'min_rate': self.min_rate, + 'max_rate': self.max_rate, + + 'leverage': self.leverage, + 'interest_rate': self.interest_rate, + 'liquidation_price': self.liquidation_price, + 'is_short': self.is_short, + 'trading_mode': self.trading_mode, + 'funding_fees': self.funding_fees, + 'open_order_id': self.open_order_id, + 'orders': orders, + } + + @staticmethod + def reset_trades() -> None: + """ + Resets all trades. Only active for backtesting mode. + """ + LocalTrade.trades = [] + LocalTrade.trades_open = [] + LocalTrade.total_profit = 0 + + def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None: + """ + Adjust the max_rate and min_rate. + """ + self.max_rate = max(current_price, self.max_rate or self.open_rate) + self.min_rate = min(current_price_low, self.min_rate or self.open_rate) + + def set_isolated_liq(self, liquidation_price: Optional[float]): + """ + Method you should use to set self.liquidation price. + Assures stop_loss is not passed the liquidation price + """ + if not liquidation_price: + return + self.liquidation_price = liquidation_price + + def _set_stop_loss(self, stop_loss: float, percent: float): + """ + Method you should use to set self.stop_loss. + Assures stop_loss is not passed the liquidation price + """ + if self.liquidation_price is not None: + if self.is_short: + sl = min(stop_loss, self.liquidation_price) + else: + sl = max(stop_loss, self.liquidation_price) + else: + sl = stop_loss + + if not self.stop_loss: + self.initial_stop_loss = sl + self.stop_loss = sl + + self.stop_loss_pct = -1 * abs(percent) + self.stoploss_last_update = datetime.utcnow() + + def adjust_stop_loss(self, current_price: float, stoploss: float, + initial: bool = False) -> None: + """ + This adjusts the stop loss to it's most recently observed setting + :param current_price: Current rate the asset is traded + :param stoploss: Stoploss as factor (sample -0.05 -> -5% below current price). + :param initial: Called to initiate stop_loss. + Skips everything if self.stop_loss is already set. + """ + if initial and not (self.stop_loss is None or self.stop_loss == 0): + # Don't modify if called with initial and nothing to do + return + + leverage = self.leverage or 1.0 + if self.is_short: + new_loss = float(current_price * (1 + abs(stoploss / leverage))) + # If trading with leverage, don't set the stoploss below the liquidation price + if self.liquidation_price: + new_loss = min(self.liquidation_price, new_loss) + else: + new_loss = float(current_price * (1 - abs(stoploss / leverage))) + # If trading with leverage, don't set the stoploss below the liquidation price + if self.liquidation_price: + new_loss = max(self.liquidation_price, new_loss) + + # no stop loss assigned yet + if self.initial_stop_loss_pct is None: + logger.debug(f"{self.pair} - Assigning new stoploss...") + self._set_stop_loss(new_loss, stoploss) + self.initial_stop_loss = new_loss + self.initial_stop_loss_pct = -1 * abs(stoploss) + + # evaluate if the stop loss needs to be updated + else: + + higher_stop = new_loss > self.stop_loss + lower_stop = new_loss < self.stop_loss + + # stop losses only walk up, never down!, + # ? But adding more to a leveraged trade would create a lower liquidation price, + # ? decreasing the minimum stoploss + if (higher_stop and not self.is_short) or (lower_stop and self.is_short): + logger.debug(f"{self.pair} - Adjusting stoploss...") + self._set_stop_loss(new_loss, stoploss) + else: + logger.debug(f"{self.pair} - Keeping current stoploss...") + + logger.debug( + f"{self.pair} - Stoploss adjusted. current_price={current_price:.8f}, " + f"open_rate={self.open_rate:.8f}, max_rate={self.max_rate or self.open_rate:.8f}, " + f"initial_stop_loss={self.initial_stop_loss:.8f}, " + f"stop_loss={self.stop_loss:.8f}. " + f"Trailing stoploss saved us: " + f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.") + + def update_trade(self, order: Order) -> None: + """ + Updates this entity with amount and actual open/close rates. + :param order: order retrieved by exchange.fetch_order() + :return: None + """ + + # Ignore open and cancelled orders + if order.status == 'open' or order.safe_price is None: + return + + logger.info(f'Updating trade (id={self.id}) ...') + + if order.ft_order_side == self.entry_side: + # Update open rate and actual amount + self.open_rate = order.safe_price + self.amount = order.safe_amount_after_fee + if self.is_open: + payment = "SELL" if self.is_short else "BUY" + logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') + self.open_order_id = None + self.recalc_trade_from_orders() + elif order.ft_order_side == self.exit_side: + if self.is_open: + payment = "BUY" if self.is_short else "SELL" + # * On margin shorts, you buy a little bit more than the amount (amount + interest) + logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') + self.close(order.safe_price) + elif order.ft_order_side == 'stoploss': + self.stoploss_order_id = None + self.close_rate_requested = self.stop_loss + self.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value + if self.is_open: + logger.info(f'{order.order_type.upper()} is hit for {self}.') + self.close(order.safe_price) + else: + raise ValueError(f'Unknown order type: {order.order_type}') + Trade.commit() + + def close(self, rate: float, *, show_msg: bool = True) -> None: + """ + Sets close_rate to the given rate, calculates total profit + and marks trade as closed + """ + self.close_rate = rate + self.close_date = self.close_date or datetime.utcnow() + self.close_profit = self.calc_profit_ratio() + self.close_profit_abs = self.calc_profit() + self.is_open = False + self.exit_order_status = 'closed' + self.open_order_id = None + if show_msg: + logger.info( + 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', + self + ) + + def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], + side: str) -> None: + """ + Update Fee parameters. Only acts once per side + """ + if self.entry_side == side and self.fee_open_currency is None: + self.fee_open_cost = fee_cost + self.fee_open_currency = fee_currency + if fee_rate is not None: + self.fee_open = fee_rate + # Assume close-fee will fall into the same fee category and take an educated guess + self.fee_close = fee_rate + elif self.exit_side == side and self.fee_close_currency is None: + self.fee_close_cost = fee_cost + self.fee_close_currency = fee_currency + if fee_rate is not None: + self.fee_close = fee_rate + + def fee_updated(self, side: str) -> bool: + """ + Verify if this side (buy / sell) has already been updated + """ + if self.entry_side == side: + return self.fee_open_currency is not None + elif self.exit_side == side: + return self.fee_close_currency is not None + else: + return False + + def update_order(self, order: Dict) -> None: + Order.update_orders(self.orders, order) + + def get_exit_order_count(self) -> int: + """ + Get amount of failed exiting orders + assumes full exits. + """ + return len([o for o in self.orders if o.ft_order_side == self.exit_side]) + + def _calc_open_trade_value(self) -> float: + """ + Calculate the open_rate including open_fee. + :return: Price in of the open trade incl. Fees + """ + open_trade = Decimal(self.amount) * Decimal(self.open_rate) + fees = open_trade * Decimal(self.fee_open) + if self.is_short: + return float(open_trade - fees) + else: + return float(open_trade + fees) + + def recalc_open_trade_value(self) -> None: + """ + Recalculate open_trade_value. + Must be called whenever open_rate, fee_open or is_short is changed. + """ + self.open_trade_value = self._calc_open_trade_value() + + def calculate_interest(self, interest_rate: Optional[float] = None) -> Decimal: + """ + :param interest_rate: interest_charge for borrowing this coin(optional). + If interest_rate is not set self.interest_rate will be used + """ + zero = Decimal(0.0) + # If nothing was borrowed + if self.trading_mode != TradingMode.MARGIN or self.has_no_leverage: + return zero + + open_date = self.open_date.replace(tzinfo=None) + now = (self.close_date or datetime.now(timezone.utc)).replace(tzinfo=None) + sec_per_hour = Decimal(3600) + total_seconds = Decimal((now - open_date).total_seconds()) + hours = total_seconds / sec_per_hour or zero + + rate = Decimal(interest_rate or self.interest_rate) + borrowed = Decimal(self.borrowed) + + return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) + + def _calc_base_close(self, amount: Decimal, rate: Optional[float] = None, + fee: Optional[float] = None) -> Decimal: + + close_trade = Decimal(amount) * Decimal(rate or self.close_rate) # type: ignore + fees = close_trade * Decimal(fee or self.fee_close) + + if self.is_short: + return close_trade + fees + else: + return close_trade - fees + + def calc_close_trade_value(self, rate: Optional[float] = None, + fee: Optional[float] = None, + interest_rate: Optional[float] = None) -> float: + """ + Calculate the close_rate including fee + :param fee: fee to use on the close rate (optional). + If rate is not set self.fee will be used + :param rate: rate to compare with (optional). + If rate is not set self.close_rate will be used + :param interest_rate: interest_charge for borrowing this coin (optional). + If interest_rate is not set self.interest_rate will be used + :return: Price in BTC of the open trade + """ + if rate is None and not self.close_rate: + return 0.0 + + amount = Decimal(self.amount) + trading_mode = self.trading_mode or TradingMode.SPOT + + if trading_mode == TradingMode.SPOT: + return float(self._calc_base_close(amount, rate, fee)) + + elif (trading_mode == TradingMode.MARGIN): + + total_interest = self.calculate_interest(interest_rate) + + if self.is_short: + amount = amount + total_interest + return float(self._calc_base_close(amount, rate, fee)) + else: + # Currency already owned for longs, no need to purchase + return float(self._calc_base_close(amount, rate, fee) - total_interest) + + elif (trading_mode == TradingMode.FUTURES): + funding_fees = self.funding_fees or 0.0 + # Positive funding_fees -> Trade has gained from fees. + # Negative funding_fees -> Trade had to pay the fees. + if self.is_short: + return float(self._calc_base_close(amount, rate, fee)) - funding_fees + else: + return float(self._calc_base_close(amount, rate, fee)) + funding_fees + else: + raise OperationalException( + f"{self.trading_mode.value} trading is not yet available using freqtrade") + + def calc_profit(self, rate: Optional[float] = None, + fee: Optional[float] = None, + interest_rate: Optional[float] = None) -> float: + """ + Calculate the absolute profit in stake currency between Close and Open trade + :param fee: fee to use on the close rate (optional). + If fee is not set self.fee will be used + :param rate: close rate to compare with (optional). + If rate is not set self.close_rate will be used + :param interest_rate: interest_charge for borrowing this coin (optional). + If interest_rate is not set self.interest_rate will be used + :return: profit in stake currency as float + """ + close_trade_value = self.calc_close_trade_value( + rate=(rate or self.close_rate), + fee=(fee or self.fee_close), + interest_rate=(interest_rate or self.interest_rate) + ) + + if self.is_short: + profit = self.open_trade_value - close_trade_value + else: + profit = close_trade_value - self.open_trade_value + return float(f"{profit:.8f}") + + def calc_profit_ratio(self, rate: Optional[float] = None, + fee: Optional[float] = None, + interest_rate: Optional[float] = None) -> float: + """ + Calculates the profit as ratio (including fee). + :param rate: rate to compare with (optional). + If rate is not set self.close_rate will be used + :param fee: fee to use on the close rate (optional). + :param interest_rate: interest_charge for borrowing this coin (optional). + If interest_rate is not set self.interest_rate will be used + :return: profit ratio as float + """ + close_trade_value = self.calc_close_trade_value( + rate=(rate or self.close_rate), + fee=(fee or self.fee_close), + interest_rate=(interest_rate or self.interest_rate) + ) + + short_close_zero = (self.is_short and close_trade_value == 0.0) + long_close_zero = (not self.is_short and self.open_trade_value == 0.0) + leverage = self.leverage or 1.0 + + if (short_close_zero or long_close_zero): + return 0.0 + else: + if self.is_short: + profit_ratio = (1 - (close_trade_value / self.open_trade_value)) * leverage + else: + profit_ratio = ((close_trade_value / self.open_trade_value) - 1) * leverage + + return float(f"{profit_ratio:.8f}") + + def recalc_trade_from_orders(self): + # We need at least 2 entry orders for averaging amounts and rates. + # TODO: this condition could probably be removed + if len(self.select_filled_orders(self.entry_side)) < 2: + self.stake_amount = self.amount * self.open_rate / self.leverage + + # Just in case, still recalc open trade value + self.recalc_open_trade_value() + return + + total_amount = 0.0 + total_stake = 0.0 + for o in self.orders: + if (o.ft_is_open or + (o.ft_order_side != self.entry_side) or + (o.status not in NON_OPEN_EXCHANGE_STATES)): + continue + + tmp_amount = o.safe_amount_after_fee + tmp_price = o.average or o.price + if o.filled is not None: + tmp_amount = o.filled + if tmp_amount > 0.0 and tmp_price is not None: + total_amount += tmp_amount + total_stake += tmp_price * tmp_amount + + if total_amount > 0: + # Leverage not updated, as we don't allow changing leverage through DCA at the moment. + self.open_rate = total_stake / total_amount + self.stake_amount = total_stake / (self.leverage or 1.0) + self.amount = total_amount + self.fee_open_cost = self.fee_open * self.stake_amount + self.recalc_open_trade_value() + if self.stop_loss_pct is not None and self.open_rate is not None: + self.adjust_stop_loss(self.open_rate, self.stop_loss_pct) + + def select_order_by_order_id(self, order_id: str) -> Optional[Order]: + """ + Finds order object by Order id. + :param order_id: Exchange order id + """ + for o in self.orders: + if o.order_id == order_id: + return o + return None + + def select_order( + self, order_side: str = None, is_open: Optional[bool] = None) -> Optional[Order]: + """ + Finds latest order for this orderside and status + :param order_side: ft_order_side of the order (either 'buy', 'sell' or 'stoploss') + :param is_open: Only search for open orders? + :return: latest Order object if it exists, else None + """ + orders = self.orders + if order_side: + orders = [o for o in self.orders if o.ft_order_side == order_side] + if is_open is not None: + orders = [o for o in orders if o.ft_is_open == is_open] + if len(orders) > 0: + return orders[-1] + else: + return None + + def select_filled_orders(self, order_side: Optional[str] = None) -> List['Order']: + """ + Finds filled orders for this orderside. + :param order_side: Side of the order (either 'buy', 'sell', or None) + :return: array of Order objects + """ + return [o for o in self.orders if ((o.ft_order_side == order_side) or (order_side is None)) + and o.ft_is_open is False and + (o.filled or 0) > 0 and + o.status in NON_OPEN_EXCHANGE_STATES] + + @property + def nr_of_successful_entries(self) -> int: + """ + Helper function to count the number of entry orders that have been filled. + :return: int count of entry orders that have been filled for this trade. + """ + + return len(self.select_filled_orders(self.entry_side)) + + @property + def nr_of_successful_exits(self) -> int: + """ + Helper function to count the number of exit orders that have been filled. + :return: int count of exit orders that have been filled for this trade. + """ + return len(self.select_filled_orders(self.exit_side)) + + @property + def nr_of_successful_buys(self) -> int: + """ + Helper function to count the number of buy orders that have been filled. + WARNING: Please use nr_of_successful_entries for short support. + :return: int count of buy orders that have been filled for this trade. + """ + + return len(self.select_filled_orders('buy')) + + @property + def nr_of_successful_sells(self) -> int: + """ + Helper function to count the number of sell orders that have been filled. + WARNING: Please use nr_of_successful_exits for short support. + :return: int count of sell orders that have been filled for this trade. + """ + return len(self.select_filled_orders('sell')) + + @property + def sell_reason(self) -> str: + """ DEPRECATED! Please use exit_reason instead.""" + return self.exit_reason + + @staticmethod + def get_trades_proxy(*, pair: str = None, is_open: bool = None, + open_date: datetime = None, close_date: datetime = None, + ) -> List['LocalTrade']: + """ + Helper function to query Trades. + Returns a List of trades, filtered on the parameters given. + In live mode, converts the filter to a database query and returns all rows + In Backtest mode, uses filters on Trade.trades to get the result. + + :return: unsorted List[Trade] + """ + + # Offline mode - without database + if is_open is not None: + if is_open: + sel_trades = LocalTrade.trades_open + else: + sel_trades = LocalTrade.trades + + else: + # Not used during backtesting, but might be used by a strategy + sel_trades = list(LocalTrade.trades + LocalTrade.trades_open) + + if pair: + sel_trades = [trade for trade in sel_trades if trade.pair == pair] + if open_date: + sel_trades = [trade for trade in sel_trades if trade.open_date > open_date] + if close_date: + sel_trades = [trade for trade in sel_trades if trade.close_date + and trade.close_date > close_date] + + return sel_trades + + @staticmethod + def close_bt_trade(trade): + LocalTrade.trades_open.remove(trade) + LocalTrade.trades.append(trade) + LocalTrade.total_profit += trade.close_profit_abs + + @staticmethod + def add_bt_trade(trade): + if trade.is_open: + LocalTrade.trades_open.append(trade) + else: + LocalTrade.trades.append(trade) + + @staticmethod + def get_open_trades() -> List[Any]: + """ + Query trades from persistence layer + """ + return Trade.get_trades_proxy(is_open=True) + + @staticmethod + def stoploss_reinitialization(desired_stoploss): + """ + Adjust initial Stoploss to desired stoploss for all open trades. + """ + for trade in Trade.get_open_trades(): + logger.info("Found open trade: %s", trade) + + # skip case if trailing-stop changed the stoploss already. + if (trade.stop_loss == trade.initial_stop_loss + and trade.initial_stop_loss_pct != desired_stoploss): + # Stoploss value got changed + + logger.info(f"Stoploss for {trade} needs adjustment...") + # Force reset of stoploss + trade.stop_loss = None + trade.initial_stop_loss_pct = None + trade.adjust_stop_loss(trade.open_rate, desired_stoploss) + logger.info(f"New stoploss: {trade.stop_loss}.") + + +class Trade(_DECL_BASE, LocalTrade): + """ + Trade database model. + Also handles updating and querying trades + + Note: Fields must be aligned with LocalTrade class + """ + __tablename__ = 'trades' + + use_db: bool = True + + id = Column(Integer, primary_key=True) + + orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined") + + exchange = Column(String(25), nullable=False) + pair = Column(String(25), nullable=False, index=True) + base_currency = Column(String(25), nullable=True) + stake_currency = Column(String(25), nullable=True) + is_open = Column(Boolean, nullable=False, default=True, index=True) + fee_open = Column(Float, nullable=False, default=0.0) + fee_open_cost = Column(Float, nullable=True) + fee_open_currency = Column(String(25), nullable=True) + fee_close = Column(Float, nullable=False, default=0.0) + fee_close_cost = Column(Float, nullable=True) + fee_close_currency = Column(String(25), nullable=True) + open_rate: float = Column(Float) + open_rate_requested = Column(Float) + # open_trade_value - calculated via _calc_open_trade_value + open_trade_value = Column(Float) + close_rate: Optional[float] = Column(Float) + close_rate_requested = Column(Float) + close_profit = Column(Float) + close_profit_abs = Column(Float) + stake_amount = Column(Float, nullable=False) + amount = Column(Float) + amount_requested = Column(Float) + open_date = Column(DateTime, nullable=False, default=datetime.utcnow) + close_date = Column(DateTime) + open_order_id = Column(String(255)) + # absolute value of the stop loss + stop_loss = Column(Float, nullable=True, default=0.0) + # percentage value of the stop loss + stop_loss_pct = Column(Float, nullable=True) + # absolute value of the initial stop loss + initial_stop_loss = Column(Float, nullable=True, default=0.0) + # percentage value of the initial stop loss + initial_stop_loss_pct = Column(Float, nullable=True) + # stoploss order id which is on exchange + stoploss_order_id = Column(String(255), nullable=True, index=True) + # last update time of the stoploss order on exchange + stoploss_last_update = Column(DateTime, nullable=True) + # absolute value of the highest reached price + max_rate = Column(Float, nullable=True, default=0.0) + # Lowest price reached + min_rate = Column(Float, nullable=True) + exit_reason = Column(String(100), nullable=True) + exit_order_status = Column(String(100), nullable=True) + strategy = Column(String(100), nullable=True) + enter_tag = Column(String(100), nullable=True) + timeframe = Column(Integer, nullable=True) + + trading_mode = Column(Enum(TradingMode), nullable=True) + + # Leverage trading properties + leverage = Column(Float, nullable=True, default=1.0) + is_short = Column(Boolean, nullable=False, default=False) + liquidation_price = Column(Float, nullable=True) + + # Margin Trading Properties + interest_rate = Column(Float, nullable=False, default=0.0) + + # Futures properties + funding_fees = Column(Float, nullable=True, default=None) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.recalc_open_trade_value() + + def delete(self) -> None: + + for order in self.orders: + Order.query.session.delete(order) + + Trade.query.session.delete(self) + Trade.commit() + + @staticmethod + def commit(): + Trade.query.session.commit() + + @staticmethod + def get_trades_proxy(*, pair: str = None, is_open: bool = None, + open_date: datetime = None, close_date: datetime = None, + ) -> List['LocalTrade']: + """ + Helper function to query Trades.j + Returns a List of trades, filtered on the parameters given. + In live mode, converts the filter to a database query and returns all rows + In Backtest mode, uses filters on Trade.trades to get the result. + + :return: unsorted List[Trade] + """ + if Trade.use_db: + trade_filter = [] + if pair: + trade_filter.append(Trade.pair == pair) + if open_date: + trade_filter.append(Trade.open_date > open_date) + if close_date: + trade_filter.append(Trade.close_date > close_date) + if is_open is not None: + trade_filter.append(Trade.is_open.is_(is_open)) + return Trade.get_trades(trade_filter).all() + else: + return LocalTrade.get_trades_proxy( + pair=pair, is_open=is_open, + open_date=open_date, + close_date=close_date + ) + + @staticmethod + def get_trades(trade_filter=None) -> Query: + """ + Helper function to query Trades using filters. + NOTE: Not supported in Backtesting. + :param trade_filter: Optional filter to apply to trades + Can be either a Filter object, or a List of filters + e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])` + e.g. `(trade_filter=Trade.id == trade_id)` + :return: unsorted query object + """ + if not Trade.use_db: + raise NotImplementedError('`Trade.get_trades()` not supported in backtesting mode.') + if trade_filter is not None: + if not isinstance(trade_filter, list): + trade_filter = [trade_filter] + return Trade.query.filter(*trade_filter) + else: + return Trade.query + + @staticmethod + def get_open_order_trades() -> List['Trade']: + """ + Returns all open trades + NOTE: Not supported in Backtesting. + """ + return Trade.get_trades(Trade.open_order_id.isnot(None)).all() + + @staticmethod + def get_open_trades_without_assigned_fees(): + """ + Returns all open trades which don't have open fees set correctly + NOTE: Not supported in Backtesting. + """ + return Trade.get_trades([Trade.fee_open_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(True), + ]).all() + + @staticmethod + def get_closed_trades_without_assigned_fees(): + """ + Returns all closed trades which don't have fees set correctly + NOTE: Not supported in Backtesting. + """ + return Trade.get_trades([Trade.fee_close_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(False), + ]).all() + + @staticmethod + def get_total_closed_profit() -> float: + """ + Retrieves total realized profit + """ + if Trade.use_db: + total_profit = Trade.query.with_entities( + func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False)).scalar() + else: + total_profit = sum( + t.close_profit_abs for t in LocalTrade.get_trades_proxy(is_open=False)) + return total_profit or 0 + + @staticmethod + def total_open_trades_stakes() -> float: + """ + Calculates total invested amount in open trades + in stake currency + """ + if Trade.use_db: + total_open_stake_amount = Trade.query.with_entities( + func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar() + else: + total_open_stake_amount = sum( + t.stake_amount for t in LocalTrade.get_trades_proxy(is_open=True)) + return total_open_stake_amount or 0 + + @staticmethod + def get_overall_performance(minutes=None) -> List[Dict[str, Any]]: + """ + Returns List of dicts containing all Trades, including profit and trade count + NOTE: Not supported in Backtesting. + """ + filters = [Trade.is_open.is_(False)] + if minutes: + start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes) + filters.append(Trade.close_date >= start_date) + pair_rates = Trade.query.with_entities( + Trade.pair, + func.sum(Trade.close_profit).label('profit_sum'), + func.sum(Trade.close_profit_abs).label('profit_sum_abs'), + func.count(Trade.pair).label('count') + ).filter(*filters)\ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum_abs')) \ + .all() + return [ + { + 'pair': pair, + 'profit_ratio': profit, + 'profit': round(profit * 100, 2), # Compatibility mode + 'profit_pct': round(profit * 100, 2), + 'profit_abs': profit_abs, + 'count': count + } + for pair, profit, profit_abs, count in pair_rates + ] + + @staticmethod + def get_enter_tag_performance(pair: Optional[str]) -> List[Dict[str, Any]]: + """ + Returns List of dicts containing all Trades, based on buy tag performance + Can either be average for all pairs or a specific pair provided + NOTE: Not supported in Backtesting. + """ + + filters = [Trade.is_open.is_(False)] + if(pair is not None): + filters.append(Trade.pair == pair) + + enter_tag_perf = Trade.query.with_entities( + Trade.enter_tag, + func.sum(Trade.close_profit).label('profit_sum'), + func.sum(Trade.close_profit_abs).label('profit_sum_abs'), + func.count(Trade.pair).label('count') + ).filter(*filters)\ + .group_by(Trade.enter_tag) \ + .order_by(desc('profit_sum_abs')) \ + .all() + + return [ + { + 'enter_tag': enter_tag if enter_tag is not None else "Other", + 'profit_ratio': profit, + 'profit_pct': round(profit * 100, 2), + 'profit_abs': profit_abs, + 'count': count + } + for enter_tag, profit, profit_abs, count in enter_tag_perf + ] + + @staticmethod + def get_exit_reason_performance(pair: Optional[str]) -> List[Dict[str, Any]]: + """ + Returns List of dicts containing all Trades, based on exit reason performance + Can either be average for all pairs or a specific pair provided + NOTE: Not supported in Backtesting. + """ + + filters = [Trade.is_open.is_(False)] + if(pair is not None): + filters.append(Trade.pair == pair) + + sell_tag_perf = Trade.query.with_entities( + Trade.exit_reason, + func.sum(Trade.close_profit).label('profit_sum'), + func.sum(Trade.close_profit_abs).label('profit_sum_abs'), + func.count(Trade.pair).label('count') + ).filter(*filters)\ + .group_by(Trade.exit_reason) \ + .order_by(desc('profit_sum_abs')) \ + .all() + + return [ + { + 'exit_reason': exit_reason if exit_reason is not None else "Other", + 'profit_ratio': profit, + 'profit_pct': round(profit * 100, 2), + 'profit_abs': profit_abs, + 'count': count + } + for exit_reason, profit, profit_abs, count in sell_tag_perf + ] + + @staticmethod + def get_mix_tag_performance(pair: Optional[str]) -> List[Dict[str, Any]]: + """ + Returns List of dicts containing all Trades, based on entry_tag + exit_reason performance + Can either be average for all pairs or a specific pair provided + NOTE: Not supported in Backtesting. + """ + + filters = [Trade.is_open.is_(False)] + if(pair is not None): + filters.append(Trade.pair == pair) + + mix_tag_perf = Trade.query.with_entities( + Trade.id, + Trade.enter_tag, + Trade.exit_reason, + func.sum(Trade.close_profit).label('profit_sum'), + func.sum(Trade.close_profit_abs).label('profit_sum_abs'), + func.count(Trade.pair).label('count') + ).filter(*filters)\ + .group_by(Trade.id) \ + .order_by(desc('profit_sum_abs')) \ + .all() + + return_list: List[Dict] = [] + for id, enter_tag, exit_reason, profit, profit_abs, count in mix_tag_perf: + enter_tag = enter_tag if enter_tag is not None else "Other" + exit_reason = exit_reason if exit_reason is not None else "Other" + + if(exit_reason is not None and enter_tag is not None): + mix_tag = enter_tag + " " + exit_reason + i = 0 + if not any(item["mix_tag"] == mix_tag for item in return_list): + return_list.append({'mix_tag': mix_tag, + 'profit': profit, + 'profit_pct': round(profit * 100, 2), + 'profit_abs': profit_abs, + 'count': count}) + else: + while i < len(return_list): + if return_list[i]["mix_tag"] == mix_tag: + return_list[i] = { + 'mix_tag': mix_tag, + 'profit': profit + return_list[i]["profit"], + 'profit_pct': round(profit + return_list[i]["profit"] * 100, 2), + 'profit_abs': profit_abs + return_list[i]["profit_abs"], + 'count': 1 + return_list[i]["count"]} + i += 1 + + return return_list + + @staticmethod + def get_best_pair(start_date: datetime = datetime.fromtimestamp(0)): + """ + Get best pair with closed trade. + NOTE: Not supported in Backtesting. + :returns: Tuple containing (pair, profit_sum) + """ + best_pair = Trade.query.with_entities( + Trade.pair, func.sum(Trade.close_profit).label('profit_sum') + ).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date)) \ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum')).first() + return best_pair diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 26efd74a9..57afbf32a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -15,10 +15,8 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, SignalTagType, SignalType, TradingMode) from freqtrade.exceptions import OperationalException, StrategyError -from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.exchange.exchange import timeframe_to_next_date -from freqtrade.persistence import PairLocks, Trade -from freqtrade.persistence.models import LocalTrade, Order +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds +from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade from freqtrade.strategy.hyper import HyperStrategyMixin from freqtrade.strategy.informative_decorator import (InformativeData, PopulateIndicators, _create_and_merge_informative_pair, From 30d6eeffd025489382170a22f779c87d72bc35ee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 May 2022 17:49:13 +0200 Subject: [PATCH 09/32] Fix migration bug --- freqtrade/persistence/migrations.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 4d29b3d49..6a77b0b9a 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -99,7 +99,10 @@ def migrate_trades_and_orders_table( liquidation_price = get_column_def(cols, 'liquidation_price', get_column_def(cols, 'isolated_liq', 'null')) # sqlite does not support literals for booleans - is_short = get_column_def(cols, 'is_short', 'false') + if engine.name == 'postgresql': + is_short = get_column_def(cols, 'is_short', 'false') + else: + is_short = get_column_def(cols, 'is_short', '0') # Margin Properties interest_rate = get_column_def(cols, 'interest_rate', '0.0') From f71b2624ab5c43a5ea8ac8d7770f506e16532a38 Mon Sep 17 00:00:00 2001 From: Luke Ingalls <45518011+lukeingalls@users.noreply.github.com> Date: Sun, 8 May 2022 10:07:22 -0700 Subject: [PATCH 10/32] then -> than --- docs/includes/pairlists.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index cec5ceb19..22a252192 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -181,7 +181,7 @@ Example to remove the first 10 pairs from the pairlist: `VolumeFilter`. !!! Note - An offset larger then the total length of the incoming pairlist will result in an empty pairlist. + An offset larger than the total length of the incoming pairlist will result in an empty pairlist. #### PerformanceFilter From df48399a901db7ab3e456e8721c8dbb27b342664 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 03:10:07 +0000 Subject: [PATCH 11/32] Bump cryptography from 37.0.1 to 37.0.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 37.0.1 to 37.0.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/37.0.1...37.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c5459a5b2..576ecddcb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ pandas-ta==0.3.14b ccxt==1.81.43 # Pin cryptography for now due to rust build errors with piwheels -cryptography==37.0.1 +cryptography==37.0.2 aiohttp==3.8.1 SQLAlchemy==1.4.36 python-telegram-bot==13.11 From 30cc8e92a1ffc0a51b32d70d9da073d0bc384d20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 03:10:15 +0000 Subject: [PATCH 12/32] Bump mkdocs-material from 8.2.12 to 8.2.14 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.2.12 to 8.2.14. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.2.12...8.2.14) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index c5a4b64b3..b26e448ea 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.2.12 +mkdocs-material==8.2.14 mdx_truly_sane_lists==1.2 pymdown-extensions==9.4 jinja2==3.1.2 From 74b309cf50b05040d9f6c3af05bcabfd423c90f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 03:10:27 +0000 Subject: [PATCH 13/32] Bump jsonschema from 4.4.0 to 4.5.1 Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.4.0 to 4.5.1. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.4.0...v4.5.1) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c5459a5b2..294a42f81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.2 cachetools==4.2.2 requests==2.27.1 urllib3==1.26.9 -jsonschema==4.4.0 +jsonschema==4.5.1 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.9 From 77a22a6b1cb2c9a86cb0b52e207dc767f4511a52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 03:10:38 +0000 Subject: [PATCH 14/32] Bump fastapi from 0.75.2 to 0.76.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.75.2 to 0.76.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.75.2...0.76.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c5459a5b2..2acdc8b78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ orjson==3.6.8 sdnotify==0.3.2 # API Server -fastapi==0.75.2 +fastapi==0.76.0 uvicorn==0.17.6 pyjwt==2.3.0 aiofiles==0.8.0 From 1ae74c1197120ad8ad67891eecd230cbbb662ca5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 03:11:12 +0000 Subject: [PATCH 15/32] Bump flake8-tidy-imports from 4.6.0 to 4.7.0 Bumps [flake8-tidy-imports](https://github.com/adamchainz/flake8-tidy-imports) from 4.6.0 to 4.7.0. - [Release notes](https://github.com/adamchainz/flake8-tidy-imports/releases) - [Changelog](https://github.com/adamchainz/flake8-tidy-imports/blob/main/HISTORY.rst) - [Commits](https://github.com/adamchainz/flake8-tidy-imports/compare/4.6.0...4.7.0) --- updated-dependencies: - dependency-name: flake8-tidy-imports dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9458be1ef..1b60d3b30 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ coveralls==3.3.1 flake8==4.0.1 -flake8-tidy-imports==4.6.0 +flake8-tidy-imports==4.7.0 mypy==0.950 pre-commit==2.18.1 pytest==7.1.2 From 69b79cd799869c4b9c3fe75015fab4bc7faccdc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 03:11:16 +0000 Subject: [PATCH 16/32] Bump types-python-dateutil from 2.8.14 to 2.8.15 Bumps [types-python-dateutil](https://github.com/python/typeshed) from 2.8.14 to 2.8.15. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-python-dateutil dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9458be1ef..7213f52b3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -26,4 +26,4 @@ types-cachetools==5.0.1 types-filelock==3.2.5 types-requests==2.27.25 types-tabulate==0.8.8 -types-python-dateutil==2.8.14 +types-python-dateutil==2.8.15 From 0756027e33e03740ecbc918077d3275fa1f16400 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 May 2022 06:32:28 +0200 Subject: [PATCH 17/32] BUmp types-python-dateutil precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1185028b9..0ce5797c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - types-filelock==3.2.5 - types-requests==2.27.25 - types-tabulate==0.8.8 - - types-python-dateutil==2.8.14 + - types-python-dateutil==2.8.15 # stages: [push] - repo: https://github.com/pycqa/isort From 5080245a73bf91c356aab5f7171423f8bf0ce6d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 04:32:48 +0000 Subject: [PATCH 18/32] Bump ccxt from 1.81.43 to 1.81.81 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.81.43 to 1.81.81. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.81.43...1.81.81) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 576ecddcb..674b1d3a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.3 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.81.43 +ccxt==1.81.81 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From 2dd655eda09973f3357ecf9eb2d5b8589240f7e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 04:52:32 +0000 Subject: [PATCH 19/32] Bump pre-commit from 2.18.1 to 2.19.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.18.1 to 2.19.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.18.1...v2.19.0) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1b60d3b30..195da7f9a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ coveralls==3.3.1 flake8==4.0.1 flake8-tidy-imports==4.7.0 mypy==0.950 -pre-commit==2.18.1 +pre-commit==2.19.0 pytest==7.1.2 pytest-asyncio==0.18.3 pytest-cov==3.0.0 From a5beacbdd0c4dc133ad0e8d1e8461f988412bb7c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 04:53:29 +0000 Subject: [PATCH 20/32] Bump types-tabulate from 0.8.8 to 0.8.9 Bumps [types-tabulate](https://github.com/python/typeshed) from 0.8.8 to 0.8.9. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-tabulate dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 42f32b01f..60f4da1a7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -25,5 +25,5 @@ nbconvert==6.5.0 types-cachetools==5.0.1 types-filelock==3.2.5 types-requests==2.27.25 -types-tabulate==0.8.8 +types-tabulate==0.8.9 types-python-dateutil==2.8.15 From 35ec657ef11149890fcd6d63080ec6b3fc4da18e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 May 2022 06:55:01 +0200 Subject: [PATCH 21/32] Bump types-tabulate==0.8.9 precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0ce5797c2..ee909185a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: - types-cachetools==5.0.1 - types-filelock==3.2.5 - types-requests==2.27.25 - - types-tabulate==0.8.8 + - types-tabulate==0.8.9 - types-python-dateutil==2.8.15 # stages: [push] From c3b0f6b64b55247ef6269a178ce46bd9cbca929a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 May 2022 07:21:10 +0200 Subject: [PATCH 22/32] Add feature shell for database conversion --- freqtrade/commands/__init__.py | 1 + freqtrade/commands/arguments.py | 27 +++++++++++++++++++-------- freqtrade/commands/cli_options.py | 5 +++++ freqtrade/commands/db_commands.py | 9 +++++++++ 4 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 freqtrade/commands/db_commands.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 129836000..1c305c3c2 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -10,6 +10,7 @@ from freqtrade.commands.arguments import Arguments from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades, start_download_data, start_list_data) +from freqtrade.commands.db_commands import start_db_convert from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index ff1d16590..6ec15dc22 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -82,7 +82,9 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "timeframe", "plot_auto_open", ] -ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version'] +ARGS_INSTALL_DB = ["db_url", "db_url_from"] + +ARGS_INSTALL_UI = ["erase_ui_only", "ui_version"] ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"] @@ -182,13 +184,14 @@ class Arguments: from freqtrade.commands import (start_backtesting, start_backtesting_show, start_convert_data, start_convert_trades, - start_create_userdir, start_download_data, start_edge, - start_hyperopt, start_hyperopt_list, start_hyperopt_show, - start_install_ui, start_list_data, start_list_exchanges, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_config, start_new_strategy, - start_plot_dataframe, start_plot_profit, start_show_trades, - start_test_pairlist, start_trading, start_webserver) + start_create_userdir, start_db_convert, start_download_data, + start_edge, start_hyperopt, start_hyperopt_list, + start_hyperopt_show, start_install_ui, start_list_data, + start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_new_config, start_new_strategy, start_plot_dataframe, + start_plot_profit, start_show_trades, start_test_pairlist, + start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -374,6 +377,14 @@ class Arguments: test_pairlist_cmd.set_defaults(func=start_test_pairlist) self._build_args(optionlist=ARGS_TEST_PAIRLIST, parser=test_pairlist_cmd) + # Add db-convert subcommand + convert_db = subparsers.add_parser( + "db-convert", + help="Migrate database to different system", + ) + convert_db.set_defaults(func=start_db_convert) + self._build_args(optionlist=ARGS_INSTALL_DB, parser=convert_db) + # Add install-ui subcommand install_ui_cmd = subparsers.add_parser( 'install-ui', diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 58e208652..24ca01760 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -106,6 +106,11 @@ AVAILABLE_CLI_OPTIONS = { f'`{constants.DEFAULT_DB_DRYRUN_URL}` for Dry Run).', metavar='PATH', ), + "db_url_from": Arg( + '--db-url-from', + help='Source db url to use when migrating database systems.', + metavar='PATH', + ), "sd_notify": Arg( '--sd-notify', help='Notify systemd service manager.', diff --git a/freqtrade/commands/db_commands.py b/freqtrade/commands/db_commands.py new file mode 100644 index 000000000..c57afa1e4 --- /dev/null +++ b/freqtrade/commands/db_commands.py @@ -0,0 +1,9 @@ +from typing import Any, Dict + +from freqtrade.configuration.config_setup import setup_utils_configuration +from freqtrade.enums.runmode import RunMode + + +def start_db_convert(args: Dict[str, Any]) -> None: + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + pass From 0958c06b84b02aca99936c8129eacbd1d068c17c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 May 2022 19:26:38 +0200 Subject: [PATCH 23/32] Implement database migration to other system --- freqtrade/commands/db_commands.py | 33 +++++++++++++++++++++++- freqtrade/configuration/configuration.py | 4 +++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/db_commands.py b/freqtrade/commands/db_commands.py index c57afa1e4..31e7cb8f2 100644 --- a/freqtrade/commands/db_commands.py +++ b/freqtrade/commands/db_commands.py @@ -1,9 +1,40 @@ +import logging from typing import Any, Dict from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.enums.runmode import RunMode +logger = logging.getLogger(__name__) + + def start_db_convert(args: Dict[str, Any]) -> None: + from sqlalchemy.orm import make_transient + + from freqtrade.persistence import Trade, init_db + from freqtrade.persistence.pairlock import PairLock + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - pass + + init_db(config['db_url'], False) + session_target = Trade._session + init_db(config['db_url_from'], False) + + # print(f"{id(sessionA)=}, {id(sessionB)=}") + trade_count = 0 + pairlock_count = 0 + for trade in Trade.get_trades(): + trade_count += 1 + make_transient(trade) + for o in trade.orders: + make_transient(o) + + session_target.add(trade) + session_target.commit() + + for pairlock in PairLock.query: + pairlock_count += 1 + make_transient(pairlock) + session_target.add(pairlock) + session_target.commit() + logger.info(f"Migrated {trade_count} Trades, and {pairlock_count} Pairlocks.") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 80df6fb3f..0346b4205 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -147,6 +147,10 @@ class Configuration: config.update({'db_url': self.args['db_url']}) logger.info('Parameter --db-url detected ...') + self._args_to_config(config, argname='db_url_from', + logstring='Parameter --db-url-from detected ...') + + if config.get('force_entry_enable', False): logger.warning('`force_entry_enable` RPC message enabled.') From c19be34e714673161cc33259157178e6460cc208 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 May 2022 19:59:15 +0200 Subject: [PATCH 24/32] Add rudimentary test for db migration --- tests/commands/test_commands.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 37eeda86a..bae623418 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -14,11 +14,13 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star start_list_exchanges, start_list_markets, start_list_strategies, start_list_timeframes, start_new_strategy, start_show_trades, start_test_pairlist, start_trading, start_webserver) +from freqtrade.commands.db_commands import start_db_convert from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException +from freqtrade.persistence.models import init_db from tests.conftest import (CURRENT_TEST_STRATEGY, create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) from tests.conftest_trades import MOCK_TRADE_COUNT @@ -1458,3 +1460,28 @@ def test_backtesting_show(mocker, testdatadir, capsys): assert sbr.call_count == 1 out, err = capsys.readouterr() assert "Pairs for Strategy" in out + + +def test_start_db_convert(mocker, fee, tmpdir, caplog): + db_src_file = Path(f"{tmpdir}/db.sqlite") + db_from = f"sqlite:///{db_src_file}" + db_target_file = Path(f"{tmpdir}/db_target.sqlite") + db_to = f"sqlite:///{db_target_file}" + args = [ + "db-convert", + "--db-url-from", + db_from, + "--db-url", + db_to, + ] + + assert not db_src_file.is_file() + init_db(db_from, False) + + create_mock_trades(fee) + assert db_src_file.is_file() + assert not db_target_file.is_file() + pargs = get_args(args) + pargs['config'] = None + start_db_convert(pargs) + assert db_target_file.is_file() From 269630e755c1483f427e4e15d1fa7f1c5f0af61c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 10 May 2022 07:01:41 +0200 Subject: [PATCH 25/32] Add preliminary documentation for database conversion --- docs/utils.md | 21 +++++++++++++++++++++ freqtrade/commands/__init__.py | 2 +- freqtrade/commands/arguments.py | 25 ++++++++++++------------- freqtrade/commands/db_commands.py | 2 +- tests/commands/test_commands.py | 8 ++++---- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 6c1b26b01..a8b8a57e3 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -554,6 +554,27 @@ Show whitelist when using a [dynamic pairlist](plugins.md#pairlists). freqtrade test-pairlist --config config.json --quote USDT BTC ``` +## Convert database + +`freqtrade convert-db` can be used to convert your database from one system to another (sqlite -> postgres, postgres -> other postgres), migrating all trades, orders and Pairlocks. + +Please refer to the [SQL cheatsheet](sql_cheatsheet.md#use-a-different-database-system) to learn about requirements for different database systems. + +``` +usage: freqtrade convert-db [-h] [--db-url PATH] [--db-url-from PATH] + +optional arguments: + -h, --help show this help message and exit + --db-url PATH Override trades database URL, this is useful in custom + deployments (default: `sqlite:///tradesv3.sqlite` for + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). + --db-url-from PATH Source db url to use when migrating database systems. +``` + +!!! Warning + Please ensure to only use this on an empty target database. Freqtrade will perform a regular migration, but may fail if entries already existed. + ## Webserver mode !!! Warning "Experimental" diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 1c305c3c2..0e637c487 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -10,7 +10,7 @@ from freqtrade.commands.arguments import Arguments from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades, start_download_data, start_list_data) -from freqtrade.commands.db_commands import start_db_convert +from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 6ec15dc22..815e28175 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -82,7 +82,7 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "timeframe", "plot_auto_open", ] -ARGS_INSTALL_DB = ["db_url", "db_url_from"] +ARGS_CONVERT_DB = ["db_url", "db_url_from"] ARGS_INSTALL_UI = ["erase_ui_only", "ui_version"] @@ -183,15 +183,14 @@ class Arguments: self._build_args(optionlist=['version'], parser=self.parser) from freqtrade.commands import (start_backtesting, start_backtesting_show, - start_convert_data, start_convert_trades, - start_create_userdir, start_db_convert, start_download_data, - start_edge, start_hyperopt, start_hyperopt_list, - start_hyperopt_show, start_install_ui, start_list_data, - start_list_exchanges, start_list_markets, - start_list_strategies, start_list_timeframes, - start_new_config, start_new_strategy, start_plot_dataframe, - start_plot_profit, start_show_trades, start_test_pairlist, - start_trading, start_webserver) + start_convert_data, start_convert_db, start_convert_trades, + start_create_userdir, start_download_data, start_edge, + start_hyperopt, start_hyperopt_list, start_hyperopt_show, + start_install_ui, start_list_data, start_list_exchanges, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_config, start_new_strategy, + start_plot_dataframe, start_plot_profit, start_show_trades, + start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -379,11 +378,11 @@ class Arguments: # Add db-convert subcommand convert_db = subparsers.add_parser( - "db-convert", + "convert-db", help="Migrate database to different system", ) - convert_db.set_defaults(func=start_db_convert) - self._build_args(optionlist=ARGS_INSTALL_DB, parser=convert_db) + convert_db.set_defaults(func=start_convert_db) + self._build_args(optionlist=ARGS_CONVERT_DB, parser=convert_db) # Add install-ui subcommand install_ui_cmd = subparsers.add_parser( diff --git a/freqtrade/commands/db_commands.py b/freqtrade/commands/db_commands.py index 31e7cb8f2..a0ad882b7 100644 --- a/freqtrade/commands/db_commands.py +++ b/freqtrade/commands/db_commands.py @@ -8,7 +8,7 @@ from freqtrade.enums.runmode import RunMode logger = logging.getLogger(__name__) -def start_db_convert(args: Dict[str, Any]) -> None: +def start_convert_db(args: Dict[str, Any]) -> None: from sqlalchemy.orm import make_transient from freqtrade.persistence import Trade, init_db diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index bae623418..9545206e2 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -14,7 +14,7 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star start_list_exchanges, start_list_markets, start_list_strategies, start_list_timeframes, start_new_strategy, start_show_trades, start_test_pairlist, start_trading, start_webserver) -from freqtrade.commands.db_commands import start_db_convert +from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration @@ -1462,13 +1462,13 @@ def test_backtesting_show(mocker, testdatadir, capsys): assert "Pairs for Strategy" in out -def test_start_db_convert(mocker, fee, tmpdir, caplog): +def test_start_convert_db(mocker, fee, tmpdir, caplog): db_src_file = Path(f"{tmpdir}/db.sqlite") db_from = f"sqlite:///{db_src_file}" db_target_file = Path(f"{tmpdir}/db_target.sqlite") db_to = f"sqlite:///{db_target_file}" args = [ - "db-convert", + "convert-db", "--db-url-from", db_from, "--db-url", @@ -1483,5 +1483,5 @@ def test_start_db_convert(mocker, fee, tmpdir, caplog): assert not db_target_file.is_file() pargs = get_args(args) pargs['config'] = None - start_db_convert(pargs) + start_convert_db(pargs) assert db_target_file.is_file() From 31cce741ac383884e7839ec99ebabf3d5ac195a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 10 May 2022 07:13:51 +0200 Subject: [PATCH 26/32] Add sequence migration --- freqtrade/commands/db_commands.py | 17 +++++++++++++++++ freqtrade/persistence/migrations.py | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/db_commands.py b/freqtrade/commands/db_commands.py index a0ad882b7..762c3c343 100644 --- a/freqtrade/commands/db_commands.py +++ b/freqtrade/commands/db_commands.py @@ -1,8 +1,12 @@ import logging from typing import Any, Dict +from sqlalchemy import func + from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.enums.runmode import RunMode +from freqtrade.persistence.migrations import set_sequence_ids +from freqtrade.persistence.trade_model import Order logger = logging.getLogger(__name__) @@ -19,6 +23,7 @@ def start_convert_db(args: Dict[str, Any]) -> None: init_db(config['db_url'], False) session_target = Trade._session init_db(config['db_url_from'], False) + logger.info("Starting db migration.") # print(f"{id(sessionA)=}, {id(sessionB)=}") trade_count = 0 @@ -30,6 +35,7 @@ def start_convert_db(args: Dict[str, Any]) -> None: make_transient(o) session_target.add(trade) + session_target.commit() for pairlock in PairLock.query: @@ -37,4 +43,15 @@ def start_convert_db(args: Dict[str, Any]) -> None: make_transient(pairlock) session_target.add(pairlock) session_target.commit() + + # Update sequences + max_trade_id = session_target.query(func.max(Trade.id)).scalar() + max_order_id = session_target.query(func.max(Order.id)).scalar() + max_pairlock_id = session_target.query(func.max(PairLock.id)).scalar() + + set_sequence_ids(session_target.get_bind(), + trade_id=max_trade_id, + order_id=max_order_id, + pairlock_id=max_pairlock_id) + logger.info(f"Migrated {trade_count} Trades, and {pairlock_count} Pairlocks.") diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 6a77b0b9a..53e35d9da 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -46,7 +46,7 @@ def get_last_sequence_ids(engine, trade_back_name, order_back_name): return order_id, trade_id -def set_sequence_ids(engine, order_id, trade_id): +def set_sequence_ids(engine, order_id, trade_id, pairlock_id=None): if engine.name == 'postgresql': with engine.begin() as connection: @@ -54,6 +54,9 @@ def set_sequence_ids(engine, order_id, trade_id): connection.execute(text(f"ALTER SEQUENCE orders_id_seq RESTART WITH {order_id}")) if trade_id: connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}")) + if pairlock_id: + connection.execute( + text(f"ALTER SEQUENCE pairlocks_id_seq RESTART WITH {pairlock_id}")) def drop_index_on_table(engine, inspector, table_bak_name): From 044afdf7afe48447973c132ade0971bab5fa2b9f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 10 May 2022 19:17:12 +0200 Subject: [PATCH 27/32] Add better test scenario --- freqtrade/configuration/configuration.py | 1 - tests/commands/test_commands.py | 7 +++++++ tests/test_persistence.py | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 0346b4205..96b585cd1 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -150,7 +150,6 @@ class Configuration: self._args_to_config(config, argname='db_url_from', logstring='Parameter --db-url-from detected ...') - if config.get('force_entry_enable', False): logger.warning('`force_entry_enable` RPC message enabled.') diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 9545206e2..0932f4362 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,5 +1,6 @@ import json import re +from datetime import datetime from io import BytesIO from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -21,6 +22,7 @@ from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.persistence.models import init_db +from freqtrade.persistence.pairlock_middleware import PairLocks from tests.conftest import (CURRENT_TEST_STRATEGY, create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) from tests.conftest_trades import MOCK_TRADE_COUNT @@ -1479,9 +1481,14 @@ def test_start_convert_db(mocker, fee, tmpdir, caplog): init_db(db_from, False) create_mock_trades(fee) + + PairLocks.timeframe = '5m' + PairLocks.lock_pair('XRP/USDT', datetime.now(), 'Random reason 125', side='long') assert db_src_file.is_file() assert not db_target_file.is_file() + pargs = get_args(args) pargs['config'] = None start_convert_db(pargs) + assert db_target_file.is_file() diff --git a/tests/test_persistence.py b/tests/test_persistence.py index b66c12086..d84415938 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1416,14 +1416,14 @@ def test_migrate_set_sequence_ids(): engine = MagicMock() engine.begin = MagicMock() engine.name = 'postgresql' - set_sequence_ids(engine, 22, 55) + set_sequence_ids(engine, 22, 55, 5) assert engine.begin.call_count == 1 engine.reset_mock() engine.begin.reset_mock() engine.name = 'somethingelse' - set_sequence_ids(engine, 22, 55) + set_sequence_ids(engine, 22, 55, 6) assert engine.begin.call_count == 0 From f374c9da705531dc6752a0b384d9664d8b052f2a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 May 2022 06:30:40 +0200 Subject: [PATCH 28/32] PR cleanup --- docs/utils.md | 2 +- freqtrade/commands/cli_options.py | 2 +- freqtrade/commands/db_commands.py | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index a8b8a57e3..9b799e5fc 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -569,7 +569,7 @@ optional arguments: deployments (default: `sqlite:///tradesv3.sqlite` for Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for Dry Run). - --db-url-from PATH Source db url to use when migrating database systems. + --db-url-from PATH Source db url to use when migrating a database. ``` !!! Warning diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 24ca01760..aac9f5713 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -108,7 +108,7 @@ AVAILABLE_CLI_OPTIONS = { ), "db_url_from": Arg( '--db-url-from', - help='Source db url to use when migrating database systems.', + help='Source db url to use when migrating a database.', metavar='PATH', ), "sd_notify": Arg( diff --git a/freqtrade/commands/db_commands.py b/freqtrade/commands/db_commands.py index 762c3c343..d93aafcb6 100644 --- a/freqtrade/commands/db_commands.py +++ b/freqtrade/commands/db_commands.py @@ -5,8 +5,6 @@ from sqlalchemy import func from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.enums.runmode import RunMode -from freqtrade.persistence.migrations import set_sequence_ids -from freqtrade.persistence.trade_model import Order logger = logging.getLogger(__name__) @@ -15,7 +13,8 @@ logger = logging.getLogger(__name__) def start_convert_db(args: Dict[str, Any]) -> None: from sqlalchemy.orm import make_transient - from freqtrade.persistence import Trade, init_db + from freqtrade.persistence import Order, Trade, init_db + from freqtrade.persistence.migrations import set_sequence_ids from freqtrade.persistence.pairlock import PairLock config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -25,7 +24,6 @@ def start_convert_db(args: Dict[str, Any]) -> None: init_db(config['db_url_from'], False) logger.info("Starting db migration.") - # print(f"{id(sessionA)=}, {id(sessionB)=}") trade_count = 0 pairlock_count = 0 for trade in Trade.get_trades(): From 8a6a6ec9111d4a42d9fd1d1f60df0133bbba7747 Mon Sep 17 00:00:00 2001 From: DJCrashdummy Date: Tue, 10 May 2022 10:33:04 +0000 Subject: [PATCH 29/32] corrected minor "typo" in formatting --- docs/includes/pairlists.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 22a252192..6acd361fe 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -44,7 +44,7 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis ```json "pairlists": [ {"method": "StaticPairList"} - ], +], ``` By default, only currently enabled pairs are allowed. From b2b503f043e0653e9e95a83af09329ffc7b8f645 Mon Sep 17 00:00:00 2001 From: DJCrashdummy Date: Wed, 11 May 2022 06:26:49 +0000 Subject: [PATCH 30/32] minor polish for explanation of --breakdown - corrected the command to fit the explanation - added a little explanation how to read the weekly & monthly breakdown --- docs/backtesting.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 75225b654..02d1a53d1 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -466,7 +466,7 @@ You can get an overview over daily / weekly or monthly results by using the `--b To visualize daily and weekly breakdowns, you can use the following: ``` bash -freqtrade backtesting --strategy MyAwesomeStrategy --breakdown day month +freqtrade backtesting --strategy MyAwesomeStrategy --breakdown day week ``` ``` output @@ -482,7 +482,7 @@ freqtrade backtesting --strategy MyAwesomeStrategy --breakdown day month ``` -The output will show a table containing the realized absolute Profit (in stake currency) for the given timeperiod, as well as wins, draws and losses that materialized (closed) on this day. +The output will show a table containing the realized absolute Profit (in stake currency) for the given timeperiod, as well as wins, draws and losses that materialized (closed) on this day. Below that there will be a second table for the summarized values of weeks indicated by the date of the closing Sunday. The same would apply to a monthly breakdown indicated by the last day of the month. ### Backtest result caching From 1fc041d0d6387712fcd69b72e1718a1820789620 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 May 2022 19:39:56 +0200 Subject: [PATCH 31/32] Fix formatting issue --- freqtrade/wallets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 9b91430e2..0c2197917 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -300,7 +300,7 @@ class Wallets: if min_stake_amount is not None and min_stake_amount > max_stake_amount: if self._log: - logger.warning("Minimum stake amount > available balance." + logger.warning("Minimum stake amount > available balance. " f"{min_stake_amount} > {max_stake_amount}") return 0 if min_stake_amount is not None and stake_amount < min_stake_amount: From c299601ece12c388a20042fc508debbd89b3ca1d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 13 May 2022 07:03:18 +0200 Subject: [PATCH 32/32] Add warning about OKX futures backtesting data --- docs/exchanges.md | 5 +++-- freqtrade/exchange/okx.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index b2759893b..50ebf9e0a 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -228,11 +228,12 @@ OKX requires a passphrase for each api key, you will therefore need to add this ``` !!! Warning - OKX only provides 300 candles per api call. Therefore, the strategy will only have a pretty low amount of data available in backtesting mode. + OKX only provides 100 candles per api call. Therefore, the strategy will only have a pretty low amount of data available in backtesting mode. -!!! Warning "Futures - position mode" +!!! Warning "Futures" OKX Futures has the concept of "position mode" - which can be Net or long/short (hedge mode). Freqtrade supports both modes - but changing the mode mid-trading is not supported and will lead to exceptions and failures to place trades. + OKX also only provides MARK candles for the past ~3 months. Backtesting futures prior to that date will therefore lead to slight deviations, as funding-fees cannot be calculated correctly without this data. ## Gate.io diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 9aeefd450..654021182 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -20,7 +20,7 @@ class Okx(Exchange): """ _ft_has: Dict = { - "ohlcv_candle_limit": 300, + "ohlcv_candle_limit": 100, "mark_ohlcv_timeframe": "4h", "funding_fee_timeframe": "8h", }