diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 8836f4f88..72938f9af 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -405,6 +405,52 @@ if self.wallets: - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above +### Additional data (Trades) + +A history of Trades can be retrieved in the strategy by querying the database. + +At the top of the file, import Trade. + +```python +from freqtrade.persistence import Trade +``` + +The following example queries for the current pair and trades from today, however other filters can easily be added. + +``` python +if self.config['runmode'] in ('live', 'dry_run'): + trades = Trade.get_trades([Trade.pair == metadata['pair'], + Trade.open_date > datetime.utcnow() - timedelta(days=1), + Trade.is_open == False, + ]).order_by(Trade.close_date).all() + # Summarize profit for this pair. + curdayprofit = sum(trade.close_profit for trade in trades) +``` + +Get amount of stake_currency currently invested in Trades: + +``` python +if self.config['runmode'] in ('live', 'dry_run'): + total_stakes = Trade.total_open_trades_stakes() +``` + +Retrieve performance per pair. +Returns a List of dicts per pair. + +``` python +if self.config['runmode'] in ('live', 'dry_run'): + performance = Trade.get_overall_performance() +``` + +Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5% (ratio of 0.015). + +``` json +{'pair': "ETH/BTC", 'profit': 0.015, 'count': 5} +``` + +!!! Warning + Trade history is not available during backtesting or hyperopt. + ### Print created dataframe To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`. diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0f5d395ff..462547d9e 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -106,7 +106,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: t.stop_loss, t.initial_stop_loss, t.strategy, t.ticker_interval ) - for t in Trade.query.all()], + for t in Trade.get_trades().all()], columns=columns) return trades diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4a494696d..7206e555f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -768,7 +768,7 @@ class FreqtradeBot: buy_timeout_threshold = arrow.utcnow().shift(minutes=-buy_timeout).datetime sell_timeout_threshold = arrow.utcnow().shift(minutes=-sell_timeout).datetime - for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): + for trade in Trade.get_open_order_trades(): try: # FIXME: Somehow the query above returns results # where the open_order_id is in fact None. diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 1850aafd9..735c740c3 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -8,17 +8,16 @@ from typing import Any, Dict, List, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, inspect) + create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import Query from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker -from sqlalchemy import func from sqlalchemy.pool import StaticPool from freqtrade import OperationalException - logger = logging.getLogger(__name__) @@ -52,9 +51,11 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: raise OperationalException(f"Given value for db_url: '{db_url}' " f"is no valid database URL! (See {_SQL_DOCS_URL})") - session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) - Trade.session = session() - Trade.query = session.query_property() + # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope + # Scoped sessions proxy requests to the appropriate thread-local session. + # We should use the scoped_session object - not a seperately initialized version + Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) + Trade.query = Trade.session.query_property() _DECL_BASE.metadata.create_all(engine) check_migrate(engine) @@ -393,6 +394,37 @@ class Trade(_DECL_BASE): profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + @staticmethod + def get_trades(trade_filter=None) -> Query: + """ + Helper function to query Trades using filters. + :param trade_filter: Optional filter to apply to trades + Can be either a Filter object, or a List of filters + e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])` + e.g. `(trade_filter=Trade.id == trade_id)` + :return: unsorted query object + """ + if trade_filter is not None: + if not isinstance(trade_filter, list): + trade_filter = [trade_filter] + return Trade.query.filter(*trade_filter) + else: + return Trade.query + + @staticmethod + def get_open_trades() -> List[Any]: + """ + Query trades from persistence layer + """ + return Trade.get_trades(Trade.is_open.is_(True)).all() + + @staticmethod + def get_open_order_trades(): + """ + Returns all open trades + """ + return Trade.get_trades(Trade.open_order_id.isnot(None)).all() + @staticmethod def total_open_trades_stakes() -> float: """ @@ -405,11 +437,38 @@ class Trade(_DECL_BASE): return total_open_stake_amount or 0 @staticmethod - def get_open_trades() -> List[Any]: + def get_overall_performance() -> List[Dict[str, Any]]: """ - Query trades from persistence layer + Returns List of dicts containing all Trades, including profit and trade count """ - return Trade.query.filter(Trade.is_open.is_(True)).all() + pair_rates = Trade.session.query( + Trade.pair, + func.sum(Trade.close_profit).label('profit_sum'), + func.count(Trade.pair).label('count') + ).filter(Trade.is_open.is_(False))\ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum')) \ + .all() + return [ + { + 'pair': pair, + 'profit': rate, + 'count': count + } + for pair, rate, count in pair_rates + ] + + @staticmethod + def get_best_pair(): + """ + Get best pair with closed trade. + """ + best_pair = Trade.session.query( + Trade.pair, func.sum(Trade.close_profit).label('profit_sum') + ).filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum')).first() + return best_pair @staticmethod def stoploss_reinitialization(desired_stoploss): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f994ac006..4aed48f74 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -9,7 +9,6 @@ from enum import Enum from typing import Dict, Any, List, Optional import arrow -import sqlalchemy as sql from numpy import mean, NAN from pandas import DataFrame @@ -154,12 +153,11 @@ class RPC: for day in range(0, timescale): profitday = today - timedelta(days=day) - trades = Trade.query \ - .filter(Trade.is_open.is_(False)) \ - .filter(Trade.close_date >= profitday)\ - .filter(Trade.close_date < (profitday + timedelta(days=1)))\ - .order_by(Trade.close_date)\ - .all() + trades = Trade.get_trades(trade_filter=[ + Trade.is_open.is_(False), + Trade.close_date >= profitday, + Trade.close_date < (profitday + timedelta(days=1)) + ]).order_by(Trade.close_date).all() curdayprofit = sum(trade.calc_profit() for trade in trades) profit_days[profitday] = { 'amount': f'{curdayprofit:.8f}', @@ -192,7 +190,7 @@ class RPC: def _rpc_trade_statistics( self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: """ Returns cumulative profit statistics """ - trades = Trade.query.order_by(Trade.id).all() + trades = Trade.get_trades().order_by(Trade.id).all() profit_all_coin = [] profit_all_perc = [] @@ -225,11 +223,7 @@ class RPC: ) profit_all_perc.append(profit_percent) - best_pair = Trade.session.query( - Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum') - ).filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')).first() + best_pair = Trade.get_best_pair() if not best_pair: raise RPCException('no closed trade') @@ -389,11 +383,8 @@ class RPC: return {'result': 'Created sell orders for all open trades.'} # Query for trade - trade = Trade.query.filter( - sql.and_( - Trade.id == trade_id, - Trade.is_open.is_(True) - ) + trade = Trade.get_trades( + trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ] ).first() if not trade: logger.warning('forcesell: Invalid argument received') @@ -423,7 +414,7 @@ class RPC: # check if valid pair # check if pair already has an open pair - trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first() + trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first() if trade: raise RPCException(f'position for {pair} already open - id: {trade.id}') @@ -432,28 +423,20 @@ class RPC: # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price): - trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first() + trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first() return trade else: return None - def _rpc_performance(self) -> List[Dict]: + def _rpc_performance(self) -> List[Dict[str, Any]]: """ Handler for performance. Shows a performance statistic from finished trades """ - - pair_rates = Trade.session.query(Trade.pair, - sql.func.sum(Trade.close_profit).label('profit_sum'), - sql.func.count(Trade.pair).label('count')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')) \ - .all() - return [ - {'pair': pair, 'profit': round(rate * 100, 2), 'count': count} - for pair, rate, count in pair_rates - ] + pair_rates = Trade.get_overall_performance() + # Round and convert to % + [x.update({'profit': round(x['profit'] * 100, 2)}) for x in pair_rates] + return pair_rates def _rpc_count(self) -> Dict[str, float]: """ Returns the number of trades running """ diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 6bd223a9b..231a1d2e2 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -35,6 +35,8 @@ def create_mock_trades(fee): fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.123, + close_rate=0.128, + close_profit=0.005, exchange='bittrex', is_open=False, open_order_id='dry_run_sell_12345' @@ -59,7 +61,7 @@ def test_init_create_session(default_conf): # Check if init create a session init(default_conf['db_url'], default_conf['dry_run']) assert hasattr(Trade, 'session') - assert 'Session' in type(Trade.session).__name__ + assert 'scoped_session' in type(Trade.session).__name__ def test_init_custom_db_url(default_conf, mocker): @@ -835,3 +837,38 @@ def test_stoploss_reinitialization(default_conf, fee): assert trade_adj.stop_loss_pct == -0.04 assert trade_adj.initial_stop_loss == 0.96 assert trade_adj.initial_stop_loss_pct == -0.04 + + +@pytest.mark.usefixtures("init_persistence") +def test_total_open_trades_stakes(fee): + + res = Trade.total_open_trades_stakes() + assert res == 0 + create_mock_trades(fee) + res = Trade.total_open_trades_stakes() + assert res == 0.002 + + +@pytest.mark.usefixtures("init_persistence") +def test_get_overall_performance(fee): + + create_mock_trades(fee) + res = Trade.get_overall_performance() + + assert len(res) == 1 + assert 'pair' in res[0] + assert 'profit' in res[0] + assert 'count' in res[0] + + +@pytest.mark.usefixtures("init_persistence") +def test_get_best_pair(fee): + + res = Trade.get_best_pair() + assert res is None + + create_mock_trades(fee) + res = Trade.get_best_pair() + assert len(res) == 2 + assert res[0] == 'ETC/BTC' + assert res[1] == 0.005