diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4e604898f..c552d8790 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -30,7 +30,7 @@ from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_signal_candles, store_backtest_stats) -from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade +from freqtrade.persistence import KeyValues, LocalTrade, Order, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -151,6 +151,7 @@ class Backtesting: LoggingMixin.show_output = True PairLocks.use_db = True Trade.use_db = True + KeyValues.use_db = True # ??? def init_backtest_detail(self): # Load detail timeframe if specified @@ -294,6 +295,8 @@ class Backtesting: Trade.use_db = False PairLocks.reset_locks() Trade.reset_trades() + KeyValues.use_db = False + KeyValues.reset_keyvalues() self.rejected_trades = 0 self.timedout_entry_orders = 0 self.timedout_exit_orders = 0 diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index ab6e2f6a5..0158f588c 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.persistence.keyvalue_middleware import KeyValues from freqtrade.persistence.models import clean_dry_run_db, cleanup_db, init_db from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import LocalTrade, Order, Trade diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/keyvalue.py new file mode 100644 index 000000000..60fa903a1 --- /dev/null +++ b/freqtrade/persistence/keyvalue.py @@ -0,0 +1,57 @@ +from datetime import datetime +from typing import Optional + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Query, relationship + +from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.persistence.base import _DECL_BASE + + +class KeyValue(_DECL_BASE): + """ + KeyValue database model + Keeps records of metadata as key/value store + for trades or global persistant values + One to many relationship with Trades: + - One trade can have many metadata entries + - One metadata entry can only be associated with one Trade + """ + __tablename__ = 'keyvalue' + # Uniqueness should be ensured over pair, order_id + # its likely that order_id is unique per Pair on some exchanges. + __table_args__ = (UniqueConstraint('ft_trade_id', 'kv_key', name="_trade_id_kv_key"),) + + id = Column(Integer, primary_key=True) + ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True, default=0) + + trade = relationship("Trade", back_populates="keyvalues") + + kv_key = Column(String(255), nullable=False) + kv_type = Column(String(25), nullable=False) + kv_value = Column(Text, nullable=False) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + updated_at = Column(DateTime, nullable=True) + + def __repr__(self): + create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT) + if self.created_at is not None else None) + update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT) + if self.updated_at is not None else None) + return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ', + f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ', + f'updated={update_time})') + + @staticmethod + def query_kv(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query: + """ + Get all keyvalues, if trade_id is not specified + return will be for generic values not tied to a trade + :param trade_id: id of the Trade + """ + key = key if key is not None else "%" + + filters = [KeyValue.ft_trade_id == trade_id if trade_id is not None else 0, + KeyValue.kv_key.ilike(key)] + + return KeyValue.query.filter(*filters) diff --git a/freqtrade/persistence/keyvalue_middleware.py b/freqtrade/persistence/keyvalue_middleware.py new file mode 100644 index 000000000..24c74610a --- /dev/null +++ b/freqtrade/persistence/keyvalue_middleware.py @@ -0,0 +1,108 @@ +import json +import logging +from datetime import datetime +from typing import Any, List, Optional + +from freqtrade.persistence.keyvalue import KeyValue + + +logger = logging.getLogger(__name__) + + +class KeyValues(): + """ + KeyValues middleware class + Abstracts the database layer away so it becomes optional - which will be necessary to support + backtesting and hyperopt in the future. + """ + + use_db = True + kvals: List[KeyValue] = [] + unserialized_types = ['bool', 'float', 'int', 'str'] + + @staticmethod + def reset_keyvalues() -> None: + """ + Resets all key-value pairs. Only active for backtesting mode. + """ + if not KeyValues.use_db: + KeyValues.kvals = [] + + @staticmethod + def get_kval(key: Optional[str] = None, trade_id: Optional[int] = None) -> List[KeyValue]: + if trade_id is None: + trade_id = 0 + + if KeyValues.use_db: + filtered_kvals = KeyValue.query_kv(trade_id=trade_id, key=key).all() + for index, kval in enumerate(filtered_kvals): + if kval.kv_type not in KeyValues.unserialized_types: + kval.kv_value = json.loads(kval.kv_value) + filtered_kvals[index] = kval + return filtered_kvals + else: + filtered_kvals = [kval for kval in KeyValues.kvals if (kval.ft_trade_id == trade_id)] + if key is not None: + filtered_kvals = [ + kval for kval in filtered_kvals if (kval.kv_key.casefold() == key.casefold())] + return filtered_kvals + + @staticmethod + def set_kval(key: str, value: Any, trade_id: Optional[int] = None) -> None: + + logger.warning(f"[set_kval] key: {key} trade_id: {trade_id} value: {value}") + value_type = type(value).__name__ + value_db = None + + if value_type not in KeyValues.unserialized_types: + try: + value_db = json.dumps(value) + except TypeError as e: + logger.warning(f"could not serialize {key} value due to {e}") + else: + value_db = str(value) + + if trade_id is None: + trade_id = 0 + + kvals = KeyValues.get_kval(key=key, trade_id=trade_id) + if kvals: + kv = kvals[0] + kv.kv_value = value + kv.updated_at = datetime.utcnow() + else: + kv = KeyValue( + ft_trade_id=trade_id, + kv_key=key, + kv_type=value_type, + kv_value=value, + created_at=datetime.utcnow() + ) + + if KeyValues.use_db and value_db is not None: + kv.kv_value = value_db + KeyValue.query.session.add(kv) + KeyValue.query.session.commit() + elif not KeyValues.use_db: + kv_index = -1 + for index, kval in enumerate(KeyValues.kvals): + if kval.ft_trade_id == trade_id and kval.kv_key == key: + kv_index = index + break + + if kv_index >= 0: + kval.kv_type = value_type + kval.value = value + kval.updated_at = datetime.utcnow() + + KeyValues.kvals[kv_index] = kval + else: + KeyValues.kvals.append(kv) + + @staticmethod + def get_all_kvals() -> List[KeyValue]: + + if KeyValues.use_db: + return KeyValue.query.all() + else: + return KeyValues.kvals diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index c31e50892..6a279ab5c 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -10,6 +10,7 @@ from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException from freqtrade.persistence.base import _DECL_BASE +from freqtrade.persistence.keyvalue import KeyValue from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Order, Trade @@ -59,6 +60,7 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None: Trade.query = Trade._session.query_property() Order.query = Trade._session.query_property() PairLock.query = Trade._session.query_property() + KeyValue.query = Trade._session.query_property() previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 358e776e3..b097f6574 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -15,6 +15,8 @@ from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.leverage import interest from freqtrade.persistence.base import _DECL_BASE +from freqtrade.persistence.keyvalue import KeyValue +from freqtrade.persistence.keyvalue_middleware import KeyValues logger = logging.getLogger(__name__) @@ -206,6 +208,7 @@ class LocalTrade(): id: int = 0 orders: List[Order] = [] + keyvalues: List[KeyValue] = [] exchange: str = '' pair: str = '' @@ -870,6 +873,12 @@ class LocalTrade(): (o.filled or 0) > 0 and o.status in NON_OPEN_EXCHANGE_STATES] + def set_kval(self, key: str, value: Any) -> None: + KeyValues.set_kval(key=key, value=value, trade_id=self.id) + + def get_kval(self, key: Optional[str]) -> List[KeyValue]: + return KeyValues.get_kval(key=key, trade_id=self.id) + @property def nr_of_successful_entries(self) -> int: """ @@ -1000,6 +1009,7 @@ class Trade(_DECL_BASE, LocalTrade): id = Column(Integer, primary_key=True) orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined") + keyvalues = relationship("KeyValue", order_by="KeyValue.id", cascade="all, delete-orphan") exchange = Column(String(25), nullable=False) pair = Column(String(25), nullable=False, index=True) @@ -1070,6 +1080,9 @@ class Trade(_DECL_BASE, LocalTrade): for order in self.orders: Order.query.session.delete(order) + for kval in self.keyvalues: + KeyValue.query.session.delete(kval) + Trade.query.session.delete(self) Trade.commit() @@ -1345,3 +1358,9 @@ class Trade(_DECL_BASE, LocalTrade): .group_by(Trade.pair) \ .order_by(desc('profit_sum')).first() return best_pair + + def set_kval(self, key: str, value: Any) -> None: + super().set_kval(key=key, value=value) + + def get_kval(self, key: Optional[str]) -> List[KeyValue]: + return super().get_kval(key=key)