From f20f5cebbe24f53660c0161b8556249d4ce0ba19 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 11:09:41 +0100 Subject: [PATCH 1/8] Move performance-calculation to persistence --- freqtrade/persistence.py | 23 ++++++++++++++++++++--- freqtrade/rpc/rpc.py | 13 +------------ tests/test_persistence.py | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 1850aafd9..6bd4b0a30 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -8,17 +8,15 @@ from typing import Any, Dict, List, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, inspect) + create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker -from sqlalchemy import func from sqlalchemy.pool import StaticPool from freqtrade import OperationalException - logger = logging.getLogger(__name__) @@ -404,6 +402,25 @@ class Trade(_DECL_BASE): .scalar() return total_open_stake_amount or 0 + @staticmethod + def get_overall_performance() -> Dict: + pair_rates = Trade.session.query( + Trade.pair, + func.sum(Trade.close_profit).label('profit_sum'), + func.count(Trade.pair).label('count') + ).filter(Trade.is_open.is_(False))\ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum')) \ + .all() + return [ + { + 'pair': pair, + 'profit': round(rate * 100, 2), + 'count': count + } + for pair, rate, count in pair_rates + ] + @staticmethod def get_open_trades() -> List[Any]: """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f994ac006..c50a7937e 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -442,18 +442,7 @@ class RPC: Handler for performance. Shows a performance statistic from finished trades """ - - pair_rates = Trade.session.query(Trade.pair, - sql.func.sum(Trade.close_profit).label('profit_sum'), - sql.func.count(Trade.pair).label('count')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')) \ - .all() - return [ - {'pair': pair, 'profit': round(rate * 100, 2), 'count': count} - for pair, rate, count in pair_rates - ] + return Trade.get_overall_performance() def _rpc_count(self) -> Dict[str, float]: """ Returns the number of trades running """ diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 6bd223a9b..8cf5f1756 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -35,6 +35,8 @@ def create_mock_trades(fee): fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.123, + close_rate=0.128, + close_profit=0.005, exchange='bittrex', is_open=False, open_order_id='dry_run_sell_12345' @@ -835,3 +837,25 @@ def test_stoploss_reinitialization(default_conf, fee): assert trade_adj.stop_loss_pct == -0.04 assert trade_adj.initial_stop_loss == 0.96 assert trade_adj.initial_stop_loss_pct == -0.04 + + +@pytest.mark.usefixtures("init_persistence") +def test_total_open_trades_stakes(fee): + + res = Trade.total_open_trades_stakes() + assert res == 0 + create_mock_trades(fee) + res = Trade.total_open_trades_stakes() + assert res == 0.002 + + +@pytest.mark.usefixtures("init_persistence") +def test_get_overall_performance(fee): + + create_mock_trades(fee) + res = Trade.get_overall_performance() + + assert len(res) == 1 + assert 'pair' in res[0] + assert 'profit' in res[0] + assert 'count' in res[0] From ab117527c9a89d68056023982ff0129c8fe71605 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 11:15:33 +0100 Subject: [PATCH 2/8] Refactor get_best_pair to persistence --- freqtrade/persistence.py | 9 +++++++++ freqtrade/rpc/rpc.py | 6 +----- tests/test_persistence.py | 13 +++++++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 6bd4b0a30..fe0b64bcc 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -421,6 +421,15 @@ class Trade(_DECL_BASE): for pair, rate, count in pair_rates ] + @staticmethod + def get_best_pair(): + best_pair = Trade.session.query( + Trade.pair, func.sum(Trade.close_profit).label('profit_sum') + ).filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum')).first() + return best_pair + @staticmethod def get_open_trades() -> List[Any]: """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c50a7937e..dc25c3743 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -225,11 +225,7 @@ class RPC: ) profit_all_perc.append(profit_percent) - best_pair = Trade.session.query( - Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum') - ).filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')).first() + best_pair = Trade.get_best_pair() if not best_pair: raise RPCException('no closed trade') diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 8cf5f1756..4aa69423e 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -859,3 +859,16 @@ def test_get_overall_performance(fee): assert 'pair' in res[0] assert 'profit' in res[0] assert 'count' in res[0] + + +@pytest.mark.usefixtures("init_persistence") +def test_get_best_pair(fee): + + res = Trade.get_best_pair() + assert res is None + + create_mock_trades(fee) + res = Trade.get_best_pair() + assert len(res) == 2 + assert res[0] == 'ETC/BTC' + assert res[1] == 0.005 From 01efebc42f27ae0250955d84bc99868ed0877a2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 13:32:07 +0100 Subject: [PATCH 3/8] Extract query to it's own function --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a8fc6bc7e..ef9a85154 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -768,7 +768,7 @@ class FreqtradeBot: buy_timeout_threshold = arrow.utcnow().shift(minutes=-buy_timeout).datetime sell_timeout_threshold = arrow.utcnow().shift(minutes=-sell_timeout).datetime - for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): + for trade in Trade.get_open_order_trades(): try: # FIXME: Somehow the query above returns results # where the open_order_id is in fact None. diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index fe0b64bcc..e527cde16 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -391,6 +391,13 @@ class Trade(_DECL_BASE): profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + @staticmethod + def get_open_order_trades(): + """ + Returns all open trades + """ + return Trade.query.filter(Trade.open_order_id.isnot(None)).all() + @staticmethod def total_open_trades_stakes() -> float: """ @@ -403,7 +410,10 @@ class Trade(_DECL_BASE): return total_open_stake_amount or 0 @staticmethod - def get_overall_performance() -> Dict: + def get_overall_performance() -> List[Dict]: + """ + Returns List of dicts containing all Trades, including profit and trade count + """ pair_rates = Trade.session.query( Trade.pair, func.sum(Trade.close_profit).label('profit_sum'), @@ -423,6 +433,9 @@ class Trade(_DECL_BASE): @staticmethod def get_best_pair(): + """ + Get best pair with closed trade. + """ best_pair = Trade.session.query( Trade.pair, func.sum(Trade.close_profit).label('profit_sum') ).filter(Trade.is_open.is_(False)) \ From 26a5800a7f9afbd8236e2d89d6c26c29556f1268 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 15:01:10 +0100 Subject: [PATCH 4/8] Extract get_trades function --- freqtrade/data/btanalysis.py | 2 +- freqtrade/persistence.py | 31 +++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0f5d395ff..462547d9e 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -106,7 +106,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: t.stop_loss, t.initial_stop_loss, t.strategy, t.ticker_interval ) - for t in Trade.query.all()], + for t in Trade.get_trades().all()], columns=columns) return trades diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index e527cde16..808e42c9a 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -11,6 +11,7 @@ from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import Query from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool @@ -391,12 +392,33 @@ class Trade(_DECL_BASE): profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + @staticmethod + def get_trades(trade_filter=None) -> Query: + """ + Helper function to query Trades using filter. + :param trade_filter: Filter to apply to trades + :return: Query object + """ + if trade_filter is not None: + if not isinstance(trade_filter, list): + trade_filter = [trade_filter] + return Trade.query.filter(*trade_filter) + else: + return Trade.query + + @staticmethod + def get_open_trades() -> List[Any]: + """ + Query trades from persistence layer + """ + return Trade.get_trades(Trade.is_open.is_(True)).all() + @staticmethod def get_open_order_trades(): """ Returns all open trades """ - return Trade.query.filter(Trade.open_order_id.isnot(None)).all() + return Trade.get_trades(Trade.open_order_id.isnot(None)).all() @staticmethod def total_open_trades_stakes() -> float: @@ -443,13 +465,6 @@ class Trade(_DECL_BASE): .order_by(desc('profit_sum')).first() return best_pair - @staticmethod - def get_open_trades() -> List[Any]: - """ - Query trades from persistence layer - """ - return Trade.query.filter(Trade.is_open.is_(True)).all() - @staticmethod def stoploss_reinitialization(desired_stoploss): """ From b37c5e4878c1900f8cb34d68438a59cf2c5c987d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 15:09:01 +0100 Subject: [PATCH 5/8] use get_trades in rpc modules --- freqtrade/persistence.py | 9 ++++++--- freqtrade/rpc/rpc.py | 25 ++++++++++--------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 808e42c9a..a15db87c7 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -395,9 +395,12 @@ class Trade(_DECL_BASE): @staticmethod def get_trades(trade_filter=None) -> Query: """ - Helper function to query Trades using filter. - :param trade_filter: Filter to apply to trades - :return: Query object + Helper function to query Trades using filters. + :param trade_filter: Optional filter to apply to trades + Can be either a Filter object, or a List of filters + e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])` + e.g. `(trade_filter=Trade.id == trade_id)` + :return: unsorted query object """ if trade_filter is not None: if not isinstance(trade_filter, list): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dc25c3743..8eecb04f9 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -9,7 +9,6 @@ from enum import Enum from typing import Dict, Any, List, Optional import arrow -import sqlalchemy as sql from numpy import mean, NAN from pandas import DataFrame @@ -154,12 +153,11 @@ class RPC: for day in range(0, timescale): profitday = today - timedelta(days=day) - trades = Trade.query \ - .filter(Trade.is_open.is_(False)) \ - .filter(Trade.close_date >= profitday)\ - .filter(Trade.close_date < (profitday + timedelta(days=1)))\ - .order_by(Trade.close_date)\ - .all() + trades = Trade.get_trades(trade_filter=[ + Trade.is_open.is_(False), + Trade.close_date >= profitday, + Trade.close_date < (profitday + timedelta(days=1)) + ]).order_by(Trade.close_date).all() curdayprofit = sum(trade.calc_profit() for trade in trades) profit_days[profitday] = { 'amount': f'{curdayprofit:.8f}', @@ -192,7 +190,7 @@ class RPC: def _rpc_trade_statistics( self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: """ Returns cumulative profit statistics """ - trades = Trade.query.order_by(Trade.id).all() + trades = Trade.get_trades().order_by(Trade.id).all() profit_all_coin = [] profit_all_perc = [] @@ -385,11 +383,8 @@ class RPC: return {'result': 'Created sell orders for all open trades.'} # Query for trade - trade = Trade.query.filter( - sql.and_( - Trade.id == trade_id, - Trade.is_open.is_(True) - ) + trade = Trade.get_trades( + trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ] ).first() if not trade: logger.warning('forcesell: Invalid argument received') @@ -419,7 +414,7 @@ class RPC: # check if valid pair # check if pair already has an open pair - trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first() + trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first() if trade: raise RPCException(f'position for {pair} already open - id: {trade.id}') @@ -428,7 +423,7 @@ class RPC: # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price): - trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first() + trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first() return trade else: return None From c2076d86a4f32901784f905f251d3b94bd1cc1a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 14:26:03 +0100 Subject: [PATCH 6/8] Use scoped_session as intended --- freqtrade/persistence.py | 8 +++++--- tests/test_persistence.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index a15db87c7..27a283378 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -51,9 +51,11 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: raise OperationalException(f"Given value for db_url: '{db_url}' " f"is no valid database URL! (See {_SQL_DOCS_URL})") - session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) - Trade.session = session() - Trade.query = session.query_property() + # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope + # Scoped sessions proxy requests to the appropriate thread-local session. + # We should use the scoped_session object - not a seperately initialized version + Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) + Trade.query = Trade.session.query_property() _DECL_BASE.metadata.create_all(engine) check_migrate(engine) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 4aa69423e..231a1d2e2 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -61,7 +61,7 @@ def test_init_create_session(default_conf): # Check if init create a session init(default_conf['db_url'], default_conf['dry_run']) assert hasattr(Trade, 'session') - assert 'Session' in type(Trade.session).__name__ + assert 'scoped_session' in type(Trade.session).__name__ def test_init_custom_db_url(default_conf, mocker): From 5ed777114837cd549b17193df6a37b3e64c41792 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 15:39:36 +0100 Subject: [PATCH 7/8] Update documentation to include get_trades fixes #1753 --- docs/strategy-customization.md | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index cef362ffd..71d6f3e8f 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -405,6 +405,52 @@ if self.wallets: - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above +### Additional data (Trades) + +A history of Trades can be retrieved in the strategy by querying the database. + +At the top of the file, import Trade. + +```python +from freqtrade.persistence import Trade +``` + +The following example queries for the current pair and trades from today, however other filters easily be added. + +``` python +if self.config['runmode'] in ('live', 'dry_run'): + trades = Trade.get_trades([Trade.pair == metadata['pair'], + Trade.open_date > datetime.utcnow() - timedelta(days=1), + Trade.is_open == False, + ]).order_by(Trade.close_date).all() + # Summarize profit for this pair. + curdayprofit = sum(trade.close_profit for trade in trades) +``` + +Get amount of stake_currency currently invested in Trades: + +``` python +if self.config['runmode'] in ('live', 'dry_run'): + total_stakes = Trade.total_open_trades_stakes() +``` + +Retrieve performance per pair. +Returns a List of dicts per pair. + +``` python +if self.config['runmode'] in ('live', 'dry_run'): + performance = Trade.get_overall_performance() +``` + +Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5%. + +``` json +{'pair': "ETH/BTC", 'profit': 1.5, 'count': 5} +``` + +!!! Warning + Trade history is not available during backtesting or hyperopt. + ### Print created dataframe To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`. From b7b1e66c6ee3c6dc4342aef73dc5c4e7c8056701 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 09:59:54 +0100 Subject: [PATCH 8/8] Convert to % as part of RPC to allow users to use unrounded ratio --- docs/strategy-customization.md | 6 +++--- freqtrade/persistence.py | 4 ++-- freqtrade/rpc/rpc.py | 7 +++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 71d6f3e8f..b3b6e3548 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -415,7 +415,7 @@ At the top of the file, import Trade. from freqtrade.persistence import Trade ``` -The following example queries for the current pair and trades from today, however other filters easily be added. +The following example queries for the current pair and trades from today, however other filters can easily be added. ``` python if self.config['runmode'] in ('live', 'dry_run'): @@ -442,10 +442,10 @@ if self.config['runmode'] in ('live', 'dry_run'): performance = Trade.get_overall_performance() ``` -Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5%. +Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5% (ratio of 0.015). ``` json -{'pair': "ETH/BTC", 'profit': 1.5, 'count': 5} +{'pair': "ETH/BTC", 'profit': 0.015, 'count': 5} ``` !!! Warning diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 27a283378..735c740c3 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -437,7 +437,7 @@ class Trade(_DECL_BASE): return total_open_stake_amount or 0 @staticmethod - def get_overall_performance() -> List[Dict]: + def get_overall_performance() -> List[Dict[str, Any]]: """ Returns List of dicts containing all Trades, including profit and trade count """ @@ -452,7 +452,7 @@ class Trade(_DECL_BASE): return [ { 'pair': pair, - 'profit': round(rate * 100, 2), + 'profit': rate, 'count': count } for pair, rate, count in pair_rates diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8eecb04f9..4aed48f74 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -428,12 +428,15 @@ class RPC: else: return None - def _rpc_performance(self) -> List[Dict]: + def _rpc_performance(self) -> List[Dict[str, Any]]: """ Handler for performance. Shows a performance statistic from finished trades """ - return Trade.get_overall_performance() + pair_rates = Trade.get_overall_performance() + # Round and convert to % + [x.update({'profit': round(x['profit'] * 100, 2)}) for x in pair_rates] + return pair_rates def _rpc_count(self) -> Dict[str, float]: """ Returns the number of trades running """