Merge pull request #2444 from freqtrade/sql_cleanup

Fix scoped_session and add Documentation for strategy

docs/strategy-customization.md

@@ -405,6 +405,52 @@ if self.wallets:
 - `get_used(asset)` - currently tied up balance (open orders)
 - `get_total(asset)` - total available balance - sum of the 2 above
 
+### Additional data (Trades)
+
+A history of Trades can be retrieved in the strategy by querying the database.
+
+At the top of the file, import Trade.
+
+```python
+from freqtrade.persistence import Trade
+```
+
+The following example queries for the current pair and trades from today, however other filters can easily be added.
+
+``` python
+if self.config['runmode'] in ('live', 'dry_run'):
+    trades = Trade.get_trades([Trade.pair == metadata['pair'],
+                               Trade.open_date > datetime.utcnow() - timedelta(days=1),
+                               Trade.is_open == False,
+                ]).order_by(Trade.close_date).all()
+    # Summarize profit for this pair.
+    curdayprofit = sum(trade.close_profit for trade in trades)
+```
+
+Get amount of stake_currency currently invested in Trades:
+
+``` python
+if self.config['runmode'] in ('live', 'dry_run'):
+    total_stakes = Trade.total_open_trades_stakes()
+```
+
+Retrieve performance per pair.
+Returns a List of dicts per pair.
+
+``` python
+if self.config['runmode'] in ('live', 'dry_run'):
+    performance = Trade.get_overall_performance()
+```
+
+Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5% (ratio of 0.015).
+
+``` json
+{'pair': "ETH/BTC", 'profit': 0.015, 'count': 5}
+```
+
+!!! Warning
+    Trade history is not available during backtesting or hyperopt.
+
 ### Print created dataframe
 
 To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`.

freqtrade/data/btanalysis.py

@@ -106,7 +106,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
                             t.stop_loss, t.initial_stop_loss,
                             t.strategy, t.ticker_interval
                             )
-                           for t in Trade.query.all()],
+                           for t in Trade.get_trades().all()],
                           columns=columns)
 
     return trades

freqtrade/freqtradebot.py

@@ -768,7 +768,7 @@ class FreqtradeBot:
         buy_timeout_threshold = arrow.utcnow().shift(minutes=-buy_timeout).datetime
         sell_timeout_threshold = arrow.utcnow().shift(minutes=-sell_timeout).datetime
 
-        for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all():
+        for trade in Trade.get_open_order_trades():
             try:
                 # FIXME: Somehow the query above returns results
                 # where the open_order_id is in fact None.

freqtrade/persistence.py

@@ -8,17 +8,16 @@ from typing import Any, Dict, List, Optional
 
 import arrow
 from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String,
-                        create_engine, inspect)
+                        create_engine, desc, func, inspect)
 from sqlalchemy.exc import NoSuchModuleError
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import Query
 from sqlalchemy.orm.scoping import scoped_session
 from sqlalchemy.orm.session import sessionmaker
-from sqlalchemy import func
 from sqlalchemy.pool import StaticPool
 
 from freqtrade import OperationalException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -52,9 +51,11 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
         raise OperationalException(f"Given value for db_url: '{db_url}' "
                                    f"is no valid database URL! (See {_SQL_DOCS_URL})")
 
-    session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
+    # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope
-    Trade.session = session()
+    # Scoped sessions proxy requests to the appropriate thread-local session.
-    Trade.query = session.query_property()
+    # We should use the scoped_session object - not a seperately initialized version
+    Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
+    Trade.query = Trade.session.query_property()
     _DECL_BASE.metadata.create_all(engine)
     check_migrate(engine)
 
@@ -393,6 +394,37 @@ class Trade(_DECL_BASE):
         profit_percent = (close_trade_price / open_trade_price) - 1
         return float(f"{profit_percent:.8f}")
 
+    @staticmethod
+    def get_trades(trade_filter=None) -> Query:
+        """
+        Helper function to query Trades using filters.
+        :param trade_filter: Optional filter to apply to trades
+                             Can be either a Filter object, or a List of filters
+                             e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
+                             e.g. `(trade_filter=Trade.id == trade_id)`
+        :return: unsorted query object
+        """
+        if trade_filter is not None:
+            if not isinstance(trade_filter, list):
+                trade_filter = [trade_filter]
+            return Trade.query.filter(*trade_filter)
+        else:
+            return Trade.query
+
+    @staticmethod
+    def get_open_trades() -> List[Any]:
+        """
+        Query trades from persistence layer
+        """
+        return Trade.get_trades(Trade.is_open.is_(True)).all()
+
+    @staticmethod
+    def get_open_order_trades():
+        """
+        Returns all open trades
+        """
+        return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
+
     @staticmethod
     def total_open_trades_stakes() -> float:
         """
@@ -405,11 +437,38 @@ class Trade(_DECL_BASE):
         return total_open_stake_amount or 0
 
     @staticmethod
-    def get_open_trades() -> List[Any]:
+    def get_overall_performance() -> List[Dict[str, Any]]:
         """
-        Query trades from persistence layer
+        Returns List of dicts containing all Trades, including profit and trade count
         """
-        return Trade.query.filter(Trade.is_open.is_(True)).all()
+        pair_rates = Trade.session.query(
+            Trade.pair,
+            func.sum(Trade.close_profit).label('profit_sum'),
+            func.count(Trade.pair).label('count')
+        ).filter(Trade.is_open.is_(False))\
+            .group_by(Trade.pair) \
+            .order_by(desc('profit_sum')) \
+            .all()
+        return [
+            {
+                'pair': pair,
+                'profit': rate,
+                'count': count
+            }
+            for pair, rate, count in pair_rates
+        ]
+
+    @staticmethod
+    def get_best_pair():
+        """
+        Get best pair with closed trade.
+        """
+        best_pair = Trade.session.query(
+            Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
+        ).filter(Trade.is_open.is_(False)) \
+            .group_by(Trade.pair) \
+            .order_by(desc('profit_sum')).first()
+        return best_pair
 
     @staticmethod
     def stoploss_reinitialization(desired_stoploss):

freqtrade/rpc/rpc.py

@@ -9,7 +9,6 @@ from enum import Enum
 from typing import Dict, Any, List, Optional
 
 import arrow
-import sqlalchemy as sql
 from numpy import mean, NAN
 from pandas import DataFrame
 
@@ -154,12 +153,11 @@ class RPC:
 
         for day in range(0, timescale):
             profitday = today - timedelta(days=day)
-            trades = Trade.query \
+            trades = Trade.get_trades(trade_filter=[
-                .filter(Trade.is_open.is_(False)) \
+                Trade.is_open.is_(False),
-                .filter(Trade.close_date >= profitday)\
+                Trade.close_date >= profitday,
-                .filter(Trade.close_date < (profitday + timedelta(days=1)))\
+                Trade.close_date < (profitday + timedelta(days=1))
-                .order_by(Trade.close_date)\
+            ]).order_by(Trade.close_date).all()
-                .all()
             curdayprofit = sum(trade.calc_profit() for trade in trades)
             profit_days[profitday] = {
                 'amount': f'{curdayprofit:.8f}',
@@ -192,7 +190,7 @@ class RPC:
     def _rpc_trade_statistics(
             self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]:
         """ Returns cumulative profit statistics """
-        trades = Trade.query.order_by(Trade.id).all()
+        trades = Trade.get_trades().order_by(Trade.id).all()
 
         profit_all_coin = []
         profit_all_perc = []
@@ -225,11 +223,7 @@ class RPC:
             )
             profit_all_perc.append(profit_percent)
 
-        best_pair = Trade.session.query(
+        best_pair = Trade.get_best_pair()
-            Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum')
-        ).filter(Trade.is_open.is_(False)) \
-            .group_by(Trade.pair) \
-            .order_by(sql.text('profit_sum DESC')).first()
 
         if not best_pair:
             raise RPCException('no closed trade')
@@ -389,11 +383,8 @@ class RPC:
             return {'result': 'Created sell orders for all open trades.'}
 
         # Query for trade
-        trade = Trade.query.filter(
+        trade = Trade.get_trades(
-            sql.and_(
+            trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ]
-                Trade.id == trade_id,
-                Trade.is_open.is_(True)
-            )
         ).first()
         if not trade:
             logger.warning('forcesell: Invalid argument received')
@@ -423,7 +414,7 @@ class RPC:
         # check if valid pair
 
         # check if pair already has an open pair
-        trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first()
+        trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first()
         if trade:
             raise RPCException(f'position for {pair} already open - id: {trade.id}')
 
@@ -432,28 +423,20 @@ class RPC:
 
         # execute buy
         if self._freqtrade.execute_buy(pair, stakeamount, price):
-            trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first()
+            trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first()
             return trade
         else:
             return None
 
-    def _rpc_performance(self) -> List[Dict]:
+    def _rpc_performance(self) -> List[Dict[str, Any]]:
         """
         Handler for performance.
         Shows a performance statistic from finished trades
         """
-
+        pair_rates = Trade.get_overall_performance()
-        pair_rates = Trade.session.query(Trade.pair,
+        # Round and convert to %
-                                         sql.func.sum(Trade.close_profit).label('profit_sum'),
+        [x.update({'profit': round(x['profit'] * 100, 2)}) for x in pair_rates]
-                                         sql.func.count(Trade.pair).label('count')) \
+        return pair_rates
-            .filter(Trade.is_open.is_(False)) \
-            .group_by(Trade.pair) \
-            .order_by(sql.text('profit_sum DESC')) \
-            .all()
-        return [
-            {'pair': pair, 'profit': round(rate * 100, 2), 'count': count}
-            for pair, rate, count in pair_rates
-        ]
 
     def _rpc_count(self) -> Dict[str, float]:
         """ Returns the number of trades running """

tests/test_persistence.py

@@ -35,6 +35,8 @@ def create_mock_trades(fee):
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_rate=0.123,
+        close_rate=0.128,
+        close_profit=0.005,
         exchange='bittrex',
         is_open=False,
         open_order_id='dry_run_sell_12345'
@@ -59,7 +61,7 @@ def test_init_create_session(default_conf):
     # Check if init create a session
     init(default_conf['db_url'], default_conf['dry_run'])
     assert hasattr(Trade, 'session')
-    assert 'Session' in type(Trade.session).__name__
+    assert 'scoped_session' in type(Trade.session).__name__
 
 
 def test_init_custom_db_url(default_conf, mocker):
@@ -835,3 +837,38 @@ def test_stoploss_reinitialization(default_conf, fee):
     assert trade_adj.stop_loss_pct == -0.04
     assert trade_adj.initial_stop_loss == 0.96
     assert trade_adj.initial_stop_loss_pct == -0.04
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_total_open_trades_stakes(fee):
+
+    res = Trade.total_open_trades_stakes()
+    assert res == 0
+    create_mock_trades(fee)
+    res = Trade.total_open_trades_stakes()
+    assert res == 0.002
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_get_overall_performance(fee):
+
+    create_mock_trades(fee)
+    res = Trade.get_overall_performance()
+
+    assert len(res) == 1
+    assert 'pair' in res[0]
+    assert 'profit' in res[0]
+    assert 'count' in res[0]
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_get_best_pair(fee):
+
+    res = Trade.get_best_pair()
+    assert res is None
+
+    create_mock_trades(fee)
+    res = Trade.get_best_pair()
+    assert len(res) == 2
+    assert res[0] == 'ETC/BTC'
+    assert res[1] == 0.005