adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge f755df2568 into e7902bffa0

Browse Source
This commit is contained in:
eSeR1805
2022-08-17 11:23:56 +02:00
committed by GitHub
parent e7902bffa0 f755df2568
commit 49ace00106
10 changed files with 339 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
freqtrade/optimize/backtesting.py
View File

@@ -30,7 +30,7 @@ from freqtrade.optimize.bt_progress import BTProgress
from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
store_backtest_signal_candles,
store_backtest_stats)
from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade
from freqtrade.persistence import CustomDataWrapper, LocalTrade, Order, PairLocks, Trade
from freqtrade.plugins.pairlistmanager import PairListManager
from freqtrade.plugins.protectionmanager import ProtectionManager
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -155,6 +155,7 @@ class Backtesting:
LoggingMixin.show_output = True
PairLocks.use_db = True
Trade.use_db = True
CustomDataWrapper.use_db = True
def init_backtest_detail(self):
# Load detail timeframe if specified
@@ -313,6 +314,8 @@ class Backtesting:
Trade.use_db = False
PairLocks.reset_locks()
Trade.reset_trades()
CustomDataWrapper.use_db = False
CustomDataWrapper.reset_custom_data()
self.rejected_trades = 0
self.timedout_entry_orders = 0
self.timedout_exit_orders = 0

1
freqtrade/persistence/__init__.py
View File

@@ -1,5 +1,6 @@
# flake8: noqa: F401
from freqtrade.persistence.custom_data_middleware import CustomDataWrapper
from freqtrade.persistence.models import init_db
from freqtrade.persistence.pairlock_middleware import PairLocks
from freqtrade.persistence.trade_model import LocalTrade, Order, Trade

57
freqtrade/persistence/custom_data.py Normal file
View File

@@ -0,0 +1,57 @@
from datetime import datetime
from typing import Optional
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Query, relationship
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.persistence.base import _DECL_BASE
class CustomData(_DECL_BASE):
"""
CustomData database model
Keeps records of metadata as key/value store
for trades or global persistant values
One to many relationship with Trades:
- One trade can have many metadata entries
- One metadata entry can only be associated with one Trade
"""
__tablename__ = 'trade_custom_data'
# Uniqueness should be ensured over pair, order_id
# its likely that order_id is unique per Pair on some exchanges.
__table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),)
id = Column(Integer, primary_key=True)
ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True, default=0)
trade = relationship("Trade", back_populates="custom_data")
cd_key = Column(String(255), nullable=False)
cd_type = Column(String(25), nullable=False)
cd_value = Column(Text, nullable=False)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
updated_at = Column(DateTime, nullable=True)
def __repr__(self):
create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT)
if self.created_at is not None else None)
update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT)
if self.updated_at is not None else None)
return (f'CustomData(id={self.id}, key={self.cd_key}, type={self.cd_type}, ' +
f'value={self.cd_value}, trade_id={self.ft_trade_id}, created={create_time}, ' +
f'updated={update_time})')
@staticmethod
def query_cd(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query:
"""
Get all CustomData, if trade_id is not specified
return will be for generic values not tied to a trade
:param trade_id: id of the Trade
"""
filters = []
filters.append(CustomData.ft_trade_id == trade_id if trade_id is not None else 0)
if key is not None:
filters.append(CustomData.cd_key.ilike(key))
return CustomData.query.filter(*filters)

113
freqtrade/persistence/custom_data_middleware.py Normal file
View File

@@ -0,0 +1,113 @@
import json
import logging
from datetime import datetime
from typing import Any, List, Optional
from freqtrade.persistence.custom_data import CustomData
logger = logging.getLogger(__name__)
class CustomDataWrapper():
"""
CustomData middleware class
Abstracts the database layer away so it becomes optional - which will be necessary to support
backtesting and hyperopt in the future.
"""
use_db = True
custom_data: List[CustomData] = []
unserialized_types = ['bool', 'float', 'int', 'str']
@staticmethod
def reset_custom_data() -> None:
"""
Resets all key-value pairs. Only active for backtesting mode.
"""
if not CustomDataWrapper.use_db:
CustomDataWrapper.custom_data = []
@staticmethod
def get_custom_data(key: Optional[str] = None,
trade_id: Optional[int] = None) -> List[CustomData]:
if trade_id is None:
trade_id = 0
if CustomDataWrapper.use_db:
filtered_custom_data = CustomData.query_cd(trade_id=trade_id, key=key).all()
for index, data_entry in enumerate(filtered_custom_data):
if data_entry.cd_type not in CustomDataWrapper.unserialized_types:
data_entry.cd_value = json.loads(data_entry.cd_value)
filtered_custom_data[index] = data_entry
return filtered_custom_data
else:
filtered_custom_data = [
data_entry for data_entry in CustomDataWrapper.custom_data
if (data_entry.ft_trade_id == trade_id)
]
if key is not None:
filtered_custom_data = [
data_entry for data_entry in filtered_custom_data
if (data_entry.cd_key.casefold() == key.casefold())
]
return filtered_custom_data
@staticmethod
def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None:
value_type = type(value).__name__
value_db = None
if value_type not in CustomDataWrapper.unserialized_types:
try:
value_db = json.dumps(value)
except TypeError as e:
logger.warning(f"could not serialize {key} value due to {e}")
else:
value_db = str(value)
if trade_id is None:
trade_id = 0
custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id)
if custom_data:
data_entry = custom_data[0]
data_entry.cd_value = value
data_entry.updated_at = datetime.utcnow()
else:
data_entry = CustomData(
ft_trade_id=trade_id,
cd_key=key,
cd_type=value_type,
cd_value=value,
created_at=datetime.utcnow()
)
if CustomDataWrapper.use_db and value_db is not None:
data_entry.cd_value = value_db
CustomData.query.session.add(data_entry)
CustomData.query.session.commit()
elif not CustomDataWrapper.use_db:
cd_index = -1
for index, data_entry in enumerate(CustomDataWrapper.custom_data):
if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key:
cd_index = index
break
if cd_index >= 0:
data_entry.cd_type = value_type
data_entry.value = value
data_entry.updated_at = datetime.utcnow()
CustomDataWrapper.custom_data[cd_index] = data_entry
else:
CustomDataWrapper.custom_data.append(data_entry)
@staticmethod
def get_all_custom_data() -> List[CustomData]:
if CustomDataWrapper.use_db:
return CustomData.query.all()
else:
return CustomDataWrapper.custom_data

3
freqtrade/persistence/models.py
View File

@@ -10,6 +10,7 @@ from sqlalchemy.pool import StaticPool
from freqtrade.exceptions import OperationalException
from freqtrade.persistence.base import _DECL_BASE
from freqtrade.persistence.custom_data import CustomData
from freqtrade.persistence.migrations import check_migrate
from freqtrade.persistence.pairlock import PairLock
from freqtrade.persistence.trade_model import Order, Trade
@@ -57,6 +58,8 @@ def init_db(db_url: str) -> None:
Trade.query = Trade._session.query_property()
Order.query = Trade._session.query_property()
PairLock.query = Trade._session.query_property()
CustomData._session = scoped_session(sessionmaker(bind=engine, autoflush=True))
CustomData.query = CustomData._session.query_property()
previous_tables = inspect(engine).get_table_names()
_DECL_BASE.metadata.create_all(engine)

20
freqtrade/persistence/trade_model.py
View File

@@ -17,6 +17,8 @@ from freqtrade.enums import ExitType, TradingMode
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.leverage import interest
from freqtrade.persistence.base import _DECL_BASE
from freqtrade.persistence.custom_data import CustomData
from freqtrade.persistence.custom_data_middleware import CustomDataWrapper
logger = logging.getLogger(__name__)
@@ -243,6 +245,7 @@ class LocalTrade():
id: int = 0
orders: List[Order] = []
custom_data: List[CustomData] = []
exchange: str = ''
pair: str = ''
@@ -939,6 +942,12 @@ class LocalTrade():
or (o.ft_is_open is True and o.status is not None)
]
def set_custom_data(self, key: str, value: Any) -> None:
CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id)
def get_custom_data(self, key: Optional[str]) -> List[CustomData]:
return CustomDataWrapper.get_custom_data(key=key, trade_id=self.id)
@property
def nr_of_successful_entries(self) -> int:
"""
@@ -1069,6 +1078,7 @@ class Trade(_DECL_BASE, LocalTrade):
id = Column(Integer, primary_key=True)
orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined")
custom_data = relationship("CustomData", order_by="CustomData.id", cascade="all, delete-orphan")
exchange = Column(String(25), nullable=False)
pair = Column(String(25), nullable=False, index=True)
@@ -1141,6 +1151,10 @@ class Trade(_DECL_BASE, LocalTrade):
for order in self.orders:
Order.query.session.delete(order)
for entry in self.custom_data:
CustomData.query.session.delete(entry)
CustomData.query.session.commit()
Trade.query.session.delete(self)
Trade.commit()
@@ -1426,6 +1440,12 @@ class Trade(_DECL_BASE, LocalTrade):
.order_by(desc('profit_sum')).first()
return best_pair
def set_custom_data(self, key: str, value: Any) -> None:
super().set_custom_data(key=key, value=value)
def get_custom_data(self, key: Optional[str]) -> List[CustomData]:
return super().get_custom_data(key=key)
@staticmethod
def get_trading_volume(start_date: datetime = datetime.fromtimestamp(0)) -> float:
"""

20
freqtrade/rpc/rpc.py
View File

@@ -829,6 +829,26 @@ class RPC:
'cancel_order_count': c_count,
}
def _rpc_list_custom_data(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]:
# Query for trade
trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first()
if trade is None:
return []
# Query custom_data
custom_data = trade.get_custom_data(key=key)
return [
{
'id': data_entry.id,
'ft_trade_id': data_entry.ft_trade_id,
'cd_key': data_entry.cd_key,
'cd_type': data_entry.cd_type,
'cd_value': data_entry.cd_value,
'created_at': data_entry.created_at,
'updated_at': data_entry.updated_at
}
for data_entry in custom_data
]
def _rpc_performance(self) -> List[Dict[str, Any]]:
"""
Handler for performance.

52
freqtrade/rpc/telegram.py
View File

@@ -191,6 +191,7 @@ class Telegram(RPCHandler):
CommandHandler('health', self._health),
CommandHandler('help', self._help),
CommandHandler('version', self._version),
CommandHandler('list_custom_data', self._list_custom_data),
]
callbacks = [
CallbackQueryHandler(self._status_table, pattern='update_status_table'),
@@ -1534,7 +1535,9 @@ class Telegram(RPCHandler):
"*/stats:* `Shows Wins / losses by Sell reason as well as "
"Avg. holding durations for buys and sells.`\n"
"*/help:* `This help message`\n"
"*/version:* `Show version`"
"*/version:* `Show version`\n"
"*/list_custom_data <trade_id> <key>:* `List custom_data for Trade ID & Key combo.`\n"
"`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
)
self._send_msg(message, parse_mode=ParseMode.MARKDOWN)
@@ -1614,6 +1617,53 @@ class Telegram(RPCHandler):
f"*Current state:* `{val['state']}`"
)
@authorized_only
def _list_custom_data(self, update: Update, context: CallbackContext) -> None:
"""
Handler for /list_custom_data <id> <key>.
List custom_data for specified trade (and key if supplied).
:param bot: telegram bot
:param update: message update
:return: None
"""
try:
if not context.args or len(context.args) == 0:
raise RPCException("Trade-id not set.")
trade_id = int(context.args[0])
key = None if len(context.args) < 2 else str(context.args[1])
results = self._rpc._rpc_list_custom_data(trade_id, key)
messages = []
if len(results) > 0:
messages.append(
'Found custom-data entr' + ('ies: ' if len(results) > 1 else 'y: ')
)
for result in results:
lines = [
f"*Key:* `{result['cd_key']}`",
f"*ID:* `{result['id']}`",
f"*Trade ID:* `{result['ft_trade_id']}`",
f"*Type:* `{result['cd_type']}`",
f"*Value:* `{result['cd_value']}`",
f"*Create Date:* `{result['created_at']}`",
f"*Update Date:* `{result['updated_at']}`"
]
# Filter empty lines using list-comprehension
messages.append("\n".join([line for line in lines if line]))
for msg in messages:
if len(msg) > MAX_MESSAGE_LENGTH:
msg = "Message dropped because length exceeds "
msg += f"maximum allowed characters: {MAX_MESSAGE_LENGTH}"
logger.warning(msg)
self._send_msg(msg)
else:
message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`"
message += f" and Key: `{key}`." if key is not None else ""
self._send_msg(message)
except RPCException as e:
self._send_msg(str(e))
def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "",
reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None:
if reload_able: