Add initial structure and wrapping.

freqtrade/optimize/backtesting.py

@@ -30,7 +30,7 @@ from freqtrade.optimize.bt_progress import BTProgress
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
                                                  store_backtest_signal_candles,
                                                  store_backtest_stats)
-from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade
+from freqtrade.persistence import KeyValues, LocalTrade, Order, PairLocks, Trade
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -151,6 +151,7 @@ class Backtesting:
         LoggingMixin.show_output = True
         PairLocks.use_db = True
         Trade.use_db = True
+        KeyValues.use_db = True  # ???
 
     def init_backtest_detail(self):
         # Load detail timeframe if specified
@@ -294,6 +295,8 @@ class Backtesting:
         Trade.use_db = False
         PairLocks.reset_locks()
         Trade.reset_trades()
+        KeyValues.use_db = False
+        KeyValues.reset_keyvalues()
         self.rejected_trades = 0
         self.timedout_entry_orders = 0
         self.timedout_exit_orders = 0

freqtrade/persistence/__init__.py

@@ -1,5 +1,6 @@
 # flake8: noqa: F401
 
+from freqtrade.persistence.keyvalue_middleware import KeyValues
 from freqtrade.persistence.models import clean_dry_run_db, cleanup_db, init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import LocalTrade, Order, Trade

freqtrade/persistence/keyvalue.py

@@ -0,0 +1,57 @@
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import Query, relationship
+
+from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.persistence.base import _DECL_BASE
+
+
+class KeyValue(_DECL_BASE):
+    """
+    KeyValue database model
+    Keeps records of metadata as key/value store
+    for trades or global persistant values
+    One to many relationship with Trades:
+      - One trade can have many metadata entries
+      - One metadata entry can only be associated with one Trade
+    """
+    __tablename__ = 'keyvalue'
+    # Uniqueness should be ensured over pair, order_id
+    # its likely that order_id is unique per Pair on some exchanges.
+    __table_args__ = (UniqueConstraint('ft_trade_id', 'kv_key', name="_trade_id_kv_key"),)
+
+    id = Column(Integer, primary_key=True)
+    ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True, default=0)
+
+    trade = relationship("Trade", back_populates="keyvalues")
+
+    kv_key = Column(String(255), nullable=False)
+    kv_type = Column(String(25), nullable=False)
+    kv_value = Column(Text, nullable=False)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(DateTime, nullable=True)
+
+    def __repr__(self):
+        create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT)
+                       if self.created_at is not None else None)
+        update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT)
+                       if self.updated_at is not None else None)
+        return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ',
+                f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ',
+                f'updated={update_time})')
+
+    @staticmethod
+    def query_kv(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query:
+        """
+        Get all keyvalues, if  trade_id is not specified
+        return will be for generic values not tied to a trade
+        :param trade_id: id of the Trade
+        """
+        key = key if key is not None else "%"
+
+        filters = [KeyValue.ft_trade_id == trade_id if trade_id is not None else 0,
+                   KeyValue.kv_key.ilike(key)]
+
+        return KeyValue.query.filter(*filters)

freqtrade/persistence/keyvalue_middleware.py

@@ -0,0 +1,108 @@
+import json
+import logging
+from datetime import datetime
+from typing import Any, List, Optional
+
+from freqtrade.persistence.keyvalue import KeyValue
+
+
+logger = logging.getLogger(__name__)
+
+
+class KeyValues():
+    """
+    KeyValues middleware class
+    Abstracts the database layer away so it becomes optional - which will be necessary to support
+    backtesting and hyperopt in the future.
+    """
+
+    use_db = True
+    kvals: List[KeyValue] = []
+    unserialized_types = ['bool', 'float', 'int', 'str']
+
+    @staticmethod
+    def reset_keyvalues() -> None:
+        """
+        Resets all key-value pairs. Only active for backtesting mode.
+        """
+        if not KeyValues.use_db:
+            KeyValues.kvals = []
+
+    @staticmethod
+    def get_kval(key: Optional[str] = None, trade_id: Optional[int] = None) -> List[KeyValue]:
+        if trade_id is None:
+            trade_id = 0
+
+        if KeyValues.use_db:
+            filtered_kvals = KeyValue.query_kv(trade_id=trade_id, key=key).all()
+            for index, kval in enumerate(filtered_kvals):
+                if kval.kv_type not in KeyValues.unserialized_types:
+                    kval.kv_value = json.loads(kval.kv_value)
+                filtered_kvals[index] = kval
+            return filtered_kvals
+        else:
+            filtered_kvals = [kval for kval in KeyValues.kvals if (kval.ft_trade_id == trade_id)]
+            if key is not None:
+                filtered_kvals = [
+                    kval for kval in filtered_kvals if (kval.kv_key.casefold() == key.casefold())]
+            return filtered_kvals
+
+    @staticmethod
+    def set_kval(key: str, value: Any, trade_id: Optional[int] = None) -> None:
+
+        logger.warning(f"[set_kval] key: {key} trade_id: {trade_id} value: {value}")
+        value_type = type(value).__name__
+        value_db = None
+
+        if value_type not in KeyValues.unserialized_types:
+            try:
+                value_db = json.dumps(value)
+            except TypeError as e:
+                logger.warning(f"could not serialize {key} value due to {e}")
+        else:
+            value_db = str(value)
+
+        if trade_id is None:
+            trade_id = 0
+
+        kvals = KeyValues.get_kval(key=key, trade_id=trade_id)
+        if kvals:
+            kv = kvals[0]
+            kv.kv_value = value
+            kv.updated_at = datetime.utcnow()
+        else:
+            kv = KeyValue(
+                ft_trade_id=trade_id,
+                kv_key=key,
+                kv_type=value_type,
+                kv_value=value,
+                created_at=datetime.utcnow()
+            )
+
+        if KeyValues.use_db and value_db is not None:
+            kv.kv_value = value_db
+            KeyValue.query.session.add(kv)
+            KeyValue.query.session.commit()
+        elif not KeyValues.use_db:
+            kv_index = -1
+            for index, kval in enumerate(KeyValues.kvals):
+                if kval.ft_trade_id == trade_id and kval.kv_key == key:
+                    kv_index = index
+                    break
+
+            if kv_index >= 0:
+                kval.kv_type = value_type
+                kval.value = value
+                kval.updated_at = datetime.utcnow()
+
+                KeyValues.kvals[kv_index] = kval
+            else:
+                KeyValues.kvals.append(kv)
+
+    @staticmethod
+    def get_all_kvals() -> List[KeyValue]:
+
+        if KeyValues.use_db:
+            return KeyValue.query.all()
+        else:
+            return KeyValues.kvals

freqtrade/persistence/models.py

@@ -10,6 +10,7 @@ from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.keyvalue import KeyValue
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
 from freqtrade.persistence.trade_model import Order, Trade
@@ -59,6 +60,7 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None:
     Trade.query = Trade._session.query_property()
     Order.query = Trade._session.query_property()
     PairLock.query = Trade._session.query_property()
+    KeyValue.query = Trade._session.query_property()
 
     previous_tables = inspect(engine).get_table_names()
     _DECL_BASE.metadata.create_all(engine)

freqtrade/persistence/trade_model.py

@@ -15,6 +15,8 @@ from freqtrade.enums import ExitType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.leverage import interest
 from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.keyvalue import KeyValue
+from freqtrade.persistence.keyvalue_middleware import KeyValues
 
 
 logger = logging.getLogger(__name__)
@@ -206,6 +208,7 @@ class LocalTrade():
     id: int = 0
 
     orders: List[Order] = []
+    keyvalues: List[KeyValue] = []
 
     exchange: str = ''
     pair: str = ''
@@ -870,6 +873,12 @@ class LocalTrade():
                 (o.filled or 0) > 0 and
                 o.status in NON_OPEN_EXCHANGE_STATES]
 
+    def set_kval(self, key: str, value: Any) -> None:
+        KeyValues.set_kval(key=key, value=value, trade_id=self.id)
+
+    def get_kval(self, key: Optional[str]) -> List[KeyValue]:
+        return KeyValues.get_kval(key=key, trade_id=self.id)
+
     @property
     def nr_of_successful_entries(self) -> int:
         """
@@ -1000,6 +1009,7 @@ class Trade(_DECL_BASE, LocalTrade):
     id = Column(Integer, primary_key=True)
 
     orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined")
+    keyvalues = relationship("KeyValue", order_by="KeyValue.id", cascade="all, delete-orphan")
 
     exchange = Column(String(25), nullable=False)
     pair = Column(String(25), nullable=False, index=True)
@@ -1070,6 +1080,9 @@ class Trade(_DECL_BASE, LocalTrade):
         for order in self.orders:
             Order.query.session.delete(order)
 
+        for kval in self.keyvalues:
+            KeyValue.query.session.delete(kval)
+
         Trade.query.session.delete(self)
         Trade.commit()
 
@@ -1345,3 +1358,9 @@ class Trade(_DECL_BASE, LocalTrade):
             .group_by(Trade.pair) \
             .order_by(desc('profit_sum')).first()
         return best_pair
+
+    def set_kval(self, key: str, value: Any) -> None:
+        super().set_kval(key=key, value=value)
+
+    def get_kval(self, key: Optional[str]) -> List[KeyValue]:
+        return super().get_kval(key=key)