Introduce Pairlocks middleware

freqtrade/freqtradebot.py

@@ -345,23 +345,23 @@ class FreqtradeBot:
         whitelist = copy.deepcopy(self.active_pair_whitelist)
         if not whitelist:
             logger.info("Active pair whitelist is empty.")
-        else:
+            return trades_created
-            # Remove pairs for currently opened trades from the whitelist
+        # Remove pairs for currently opened trades from the whitelist
-            for trade in Trade.get_open_trades():
+        for trade in Trade.get_open_trades():
-                if trade.pair in whitelist:
+            if trade.pair in whitelist:
-                    whitelist.remove(trade.pair)
+                whitelist.remove(trade.pair)
-                    logger.debug('Ignoring %s in pair whitelist', trade.pair)
+                logger.debug('Ignoring %s in pair whitelist', trade.pair)
 
-            if not whitelist:
+        if not whitelist:
-                logger.info("No currency pair in active pair whitelist, "
+            logger.info("No currency pair in active pair whitelist, "
-                            "but checking to sell open trades.")
+                        "but checking to sell open trades.")
-            else:
+            return trades_created
-                # Create entity and execute trade for each pair from whitelist
+        # Create entity and execute trade for each pair from whitelist
-                for pair in whitelist:
+        for pair in whitelist:
-                    try:
+            try:
-                        trades_created += self.create_trade(pair)
+                trades_created += self.create_trade(pair)
-                    except DependencyException as exception:
+            except DependencyException as exception:
-                        logger.warning('Unable to create trade for %s: %s', pair, exception)
+                logger.warning('Unable to create trade for %s: %s', pair, exception)
 
                 if not trades_created:
                     logger.debug("Found no buy signals for whitelisted currencies. "

freqtrade/persistence/__init__.py

@@ -1,4 +1,4 @@
 # flake8: noqa: F401
 
-from freqtrade.persistence.models import (Order, PairLock, Trade, clean_dry_run_db, cleanup_db,
+from freqtrade.persistence.models import Order, Trade, clean_dry_run_db, cleanup_db, init_db
-                                          init_db)
+from freqtrade.persistence.pairlock_middleware import PairLocks

freqtrade/persistence/models.py

@@ -684,19 +684,7 @@ class PairLock(_DECL_BASE):
                 f'lock_end_time={lock_end_time})')
 
     @staticmethod
-    def lock_pair(pair: str, until: datetime, reason: str = None) -> None:
+    def query_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> Query:
-        lock = PairLock(
-            pair=pair,
-            lock_time=datetime.now(timezone.utc),
-            lock_end_time=until,
-            reason=reason,
-            active=True
-        )
-        PairLock.session.add(lock)
-        PairLock.session.flush()
-
-    @staticmethod
-    def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List['PairLock']:
         """
         Get all locks for this pair
         :param pair: Pair to check for. Returns all current locks if pair is empty
@@ -713,41 +701,7 @@ class PairLock(_DECL_BASE):
             filters.append(PairLock.pair == pair)
         return PairLock.query.filter(
             *filters
-        ).all()
+        )
-
-    @staticmethod
-    def unlock_pair(pair: str, now: Optional[datetime] = None) -> None:
-        """
-        Release all locks for this pair.
-        :param pair: Pair to unlock
-        :param now: Datetime object (generated via datetime.now(timezone.utc)).
-            defaults to datetime.utcnow()
-        """
-        if not now:
-            now = datetime.now(timezone.utc)
-
-        logger.info(f"Releasing all locks for {pair}.")
-        locks = PairLock.get_pair_locks(pair, now)
-        for lock in locks:
-            lock.active = False
-        PairLock.session.flush()
-
-    @staticmethod
-    def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool:
-        """
-        :param pair: Pair to check for
-        :param now: Datetime object (generated via datetime.now(timezone.utc)).
-            defaults to datetime.utcnow()
-        """
-        if not now:
-            now = datetime.now(timezone.utc)
-
-        return PairLock.query.filter(
-            PairLock.pair == pair,
-            func.datetime(PairLock.lock_end_time) >= now,
-            # Only active locks
-            PairLock.active.is_(True),
-        ).first() is not None
 
     def to_json(self) -> Dict[str, Any]:
         return {

freqtrade/persistence/pairlock_middleware.py

@@ -0,0 +1,97 @@
+
+
+import logging
+from datetime import datetime, timezone
+from typing import List, Optional
+
+from freqtrade.persistence.models import PairLock
+
+
+logger = logging.getLogger(__name__)
+
+
+class PairLocks():
+    """
+    Pairlocks intermediate class
+
+    """
+
+    use_db = True
+    locks: List[PairLock] = []
+
+    @staticmethod
+    def lock_pair(pair: str, until: datetime, reason: str = None) -> None:
+        lock = PairLock(
+            pair=pair,
+            lock_time=datetime.now(timezone.utc),
+            lock_end_time=until,
+            reason=reason,
+            active=True
+        )
+        if PairLocks.use_db:
+            PairLock.session.add(lock)
+            PairLock.session.flush()
+        else:
+            PairLocks.locks.append(lock)
+
+    @staticmethod
+    def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]:
+        """
+        Get all currently active locks for this pair
+        :param pair: Pair to check for. Returns all current locks if pair is empty
+        :param now: Datetime object (generated via datetime.now(timezone.utc)).
+                    defaults to datetime.utcnow()
+        """
+        if not now:
+            now = datetime.now(timezone.utc)
+
+        if PairLocks.use_db:
+            return PairLock.query_pair_locks(pair, now).all()
+        else:
+            locks = [lock for lock in PairLocks.locks if (
+                lock.lock_end_time > now
+                and lock.active is True
+                and (pair is None or lock.pair == pair)
+            )]
+            return locks
+
+    @staticmethod
+    def unlock_pair(pair: str, now: Optional[datetime] = None) -> None:
+        """
+        Release all locks for this pair.
+        :param pair: Pair to unlock
+        :param now: Datetime object (generated via datetime.now(timezone.utc)).
+            defaults to datetime.now(timezone.utc)
+        """
+        if not now:
+            now = datetime.now(timezone.utc)
+
+        logger.info(f"Releasing all locks for {pair}.")
+        locks = PairLocks.get_pair_locks(pair, now)
+        for lock in locks:
+            lock.active = False
+        if PairLocks.use_db:
+            PairLock.session.flush()
+
+    @staticmethod
+    def is_global_lock(now: Optional[datetime] = None) -> bool:
+        """
+        :param now: Datetime object (generated via datetime.now(timezone.utc)).
+            defaults to datetime.now(timezone.utc)
+        """
+        if not now:
+            now = datetime.now(timezone.utc)
+
+        return len(PairLocks.get_pair_locks('*', now)) > 0
+
+    @staticmethod
+    def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool:
+        """
+        :param pair: Pair to check for
+        :param now: Datetime object (generated via datetime.now(timezone.utc)).
+            defaults to datetime.now(timezone.utc)
+        """
+        if not now:
+            now = datetime.now(timezone.utc)
+
+        return len(PairLocks.get_pair_locks(pair, now)) > 0 or PairLocks.is_global_lock(now)

freqtrade/rpc/rpc.py

@@ -19,7 +19,7 @@ from freqtrade.exceptions import ExchangeError, PricingError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.loggers import bufferHandler
 from freqtrade.misc import shorten_date
-from freqtrade.persistence import PairLock, Trade
+from freqtrade.persistence import PairLocks, Trade
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
 from freqtrade.state import State
 from freqtrade.strategy.interface import SellType
@@ -604,7 +604,7 @@ class RPC:
         if self._freqtrade.state != State.RUNNING:
             raise RPCException('trader is not running')
 
-        locks = PairLock.get_pair_locks(None)
+        locks = PairLocks.get_pair_locks(None)
         return {
             'lock_count': len(locks),
             'locks': [lock.to_json() for lock in locks]

freqtrade/strategy/interface.py

@@ -17,7 +17,7 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.exceptions import OperationalException, StrategyError
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.exchange.exchange import timeframe_to_next_date
-from freqtrade.persistence import PairLock, Trade
+from freqtrade.persistence import PairLocks, Trade
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.wallets import Wallets
 
@@ -288,7 +288,7 @@ class IStrategy(ABC):
                 Needs to be timezone aware `datetime.now(timezone.utc)`
         :param reason: Optional string explaining why the pair was locked.
         """
-        PairLock.lock_pair(pair, until, reason)
+        PairLocks.lock_pair(pair, until, reason)
 
     def unlock_pair(self, pair: str) -> None:
         """
@@ -297,7 +297,7 @@ class IStrategy(ABC):
         manually from within the strategy, to allow an easy way to unlock pairs.
         :param pair: Unlock pair to allow trading again
         """
-        PairLock.unlock_pair(pair, datetime.now(timezone.utc))
+        PairLocks.unlock_pair(pair, datetime.now(timezone.utc))
 
     def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool:
         """
@@ -312,10 +312,10 @@ class IStrategy(ABC):
 
         if not candle_date:
             # Simple call ...
-            return PairLock.is_pair_locked(pair, candle_date)
+            return PairLocks.is_pair_locked(pair, candle_date)
         else:
             lock_time = timeframe_to_next_date(self.timeframe, candle_date)
-            return PairLock.is_pair_locked(pair, lock_time)
+            return PairLocks.is_pair_locked(pair, lock_time)
 
     def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """

tests/pairlist/test_pairlocks.py

@@ -0,0 +1,81 @@
+from datetime import datetime, timedelta, timezone
+
+import arrow
+import pytest
+
+from freqtrade.persistence import PairLocks
+from freqtrade.persistence.models import PairLock
+
+
+@pytest.mark.parametrize('use_db', (False, True))
+@pytest.mark.usefixtures("init_persistence")
+def test_PairLocks(use_db):
+    # No lock should be present
+    if use_db:
+        assert len(PairLock.query.all()) == 0
+    else:
+        PairLocks.use_db = False
+
+    assert PairLocks.use_db == use_db
+
+    pair = 'ETH/BTC'
+    assert not PairLocks.is_pair_locked(pair)
+    PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
+    # ETH/BTC locked for 4 minutes
+    assert PairLocks.is_pair_locked(pair)
+
+    # XRP/BTC should not be locked now
+    pair = 'XRP/BTC'
+    assert not PairLocks.is_pair_locked(pair)
+    # Unlocking a pair that's not locked should not raise an error
+    PairLocks.unlock_pair(pair)
+
+    PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
+    assert PairLocks.is_pair_locked(pair)
+
+    # Get both locks from above
+    locks = PairLocks.get_pair_locks(None)
+    assert len(locks) == 2
+
+    # Unlock original pair
+    pair = 'ETH/BTC'
+    PairLocks.unlock_pair(pair)
+    assert not PairLocks.is_pair_locked(pair)
+    assert not PairLocks.is_global_lock()
+
+    pair = 'BTC/USDT'
+    # Lock until 14:30
+    lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc)
+    PairLocks.lock_pair(pair, lock_time)
+
+    assert not PairLocks.is_pair_locked(pair)
+    assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-10))
+    assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-10))
+    assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
+    assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50))
+
+    # Should not be locked after time expired
+    assert not PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=10))
+
+    locks = PairLocks.get_pair_locks(pair, lock_time + timedelta(minutes=-2))
+    assert len(locks) == 1
+    assert 'PairLock' in str(locks[0])
+
+    # Unlock all
+    PairLocks.unlock_pair(pair, lock_time + timedelta(minutes=-2))
+    assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50))
+
+    # Global lock
+    PairLocks.lock_pair('*', lock_time)
+    assert PairLocks.is_global_lock(lock_time + timedelta(minutes=-50))
+    # Global lock also locks every pair seperately
+    assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
+    assert PairLocks.is_pair_locked('XRP/USDT', lock_time + timedelta(minutes=-50))
+
+    if use_db:
+        assert len(PairLock.query.all()) > 0
+    else:
+        # Nothing was pushed to the database
+        assert len(PairLock.query.all()) == 0
+    # Reset use-db variable
+    PairLocks.use_db = True

tests/rpc/test_rpc_apiserver.py

@@ -12,7 +12,7 @@ from requests.auth import _basic_auth_str
 
 from freqtrade.__init__ import __version__
 from freqtrade.loggers import setup_logging, setup_logging_pre
-from freqtrade.persistence import PairLock, Trade
+from freqtrade.persistence import PairLocks, Trade
 from freqtrade.rpc.api_server import BASE_URI, ApiServer
 from freqtrade.state import State
 from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, patch_get_signal
@@ -339,8 +339,8 @@ def test_api_locks(botclient):
     assert rc.json['lock_count'] == 0
     assert rc.json['lock_count'] == len(rc.json['locks'])
 
-    PairLock.lock_pair('ETH/BTC', datetime.utcnow() + timedelta(minutes=4), 'randreason')
+    PairLocks.lock_pair('ETH/BTC', datetime.utcnow() + timedelta(minutes=4), 'randreason')
-    PairLock.lock_pair('XRP/BTC', datetime.utcnow() + timedelta(minutes=20), 'deadbeef')
+    PairLocks.lock_pair('XRP/BTC', datetime.utcnow() + timedelta(minutes=20), 'deadbeef')
 
     rc = client_get(client, f"{BASE_URI}/locks")
     assert_response(rc)

tests/rpc/test_rpc_telegram.py

@@ -18,7 +18,7 @@ from freqtrade.constants import CANCEL_REASON
 from freqtrade.edge import PairInfo
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.loggers import setup_logging
-from freqtrade.persistence import PairLock, Trade
+from freqtrade.persistence import PairLocks, Trade
 from freqtrade.rpc import RPCMessageType
 from freqtrade.rpc.telegram import Telegram, authorized_only
 from freqtrade.state import State
@@ -1047,8 +1047,8 @@ def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) -> None
     msg_mock.reset_mock()
     freqtradebot.state = State.RUNNING
 
-    PairLock.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason')
+    PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason')
-    PairLock.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef')
+    PairLocks.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef')
 
     telegram._locks(update=update, context=MagicMock())
 

tests/strategy/test_interface.py

@@ -11,7 +11,7 @@ from freqtrade.configuration import TimeRange
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history import load_data
 from freqtrade.exceptions import StrategyError
-from freqtrade.persistence import PairLock, Trade
+from freqtrade.persistence import PairLocks, Trade
 from freqtrade.resolvers import StrategyResolver
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from tests.conftest import log_has, log_has_re
@@ -364,7 +364,7 @@ def test_is_pair_locked(default_conf):
     default_conf.update({'strategy': 'DefaultStrategy'})
     strategy = StrategyResolver.load_strategy(default_conf)
     # No lock should be present
-    assert len(PairLock.query.all()) == 0
+    assert len(PairLocks.get_pair_locks(None)) == 0
 
     pair = 'ETH/BTC'
     assert not strategy.is_pair_locked(pair)

tests/test_freqtradebot.py

@@ -15,7 +15,8 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie
                                   InvalidOrderException, OperationalException, PricingError,
                                   TemporaryError)
 from freqtrade.freqtradebot import FreqtradeBot
-from freqtrade.persistence import Order, PairLock, Trade
+from freqtrade.persistence import Order, PairLocks, Trade
+from freqtrade.persistence.models import PairLock
 from freqtrade.rpc import RPCMessageType
 from freqtrade.state import RunMode, State
 from freqtrade.strategy.interface import SellCheckTuple, SellType

tests/test_persistence.py

@@ -1,6 +1,5 @@
 # pragma pylint: disable=missing-docstring, C0103
 import logging
-from datetime import datetime, timedelta, timezone
 from unittest.mock import MagicMock
 
 import arrow
@@ -9,7 +8,7 @@ from sqlalchemy import create_engine
 
 from freqtrade import constants
 from freqtrade.exceptions import DependencyException, OperationalException
-from freqtrade.persistence import Order, PairLock, Trade, clean_dry_run_db, init_db
+from freqtrade.persistence import Order, Trade, clean_dry_run_db, init_db
 from tests.conftest import create_mock_trades, log_has, log_has_re
 
 
@@ -1159,49 +1158,3 @@ def test_select_order(fee):
     assert order.ft_order_side == 'stoploss'
     order = trades[4].select_order('sell', False)
     assert order is None
-
-
-@pytest.mark.usefixtures("init_persistence")
-def test_PairLock(default_conf):
-    # No lock should be present
-    assert len(PairLock.query.all()) == 0
-
-    pair = 'ETH/BTC'
-    assert not PairLock.is_pair_locked(pair)
-    PairLock.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
-    # ETH/BTC locked for 4 minutes
-    assert PairLock.is_pair_locked(pair)
-
-    # XRP/BTC should not be locked now
-    pair = 'XRP/BTC'
-    assert not PairLock.is_pair_locked(pair)
-    # Unlocking a pair that's not locked should not raise an error
-    PairLock.unlock_pair(pair)
-
-    PairLock.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
-    assert PairLock.is_pair_locked(pair)
-
-    # Get both locks from above
-    locks = PairLock.get_pair_locks(None)
-    assert len(locks) == 2
-
-    # Unlock original pair
-    pair = 'ETH/BTC'
-    PairLock.unlock_pair(pair)
-    assert not PairLock.is_pair_locked(pair)
-
-    pair = 'BTC/USDT'
-    # Lock until 14:30
-    lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc)
-    PairLock.lock_pair(pair, lock_time)
-
-    assert not PairLock.is_pair_locked(pair)
-    assert PairLock.is_pair_locked(pair, lock_time + timedelta(minutes=-10))
-    assert PairLock.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
-
-    # Should not be locked after time expired
-    assert not PairLock.is_pair_locked(pair, lock_time + timedelta(minutes=10))
-
-    locks = PairLock.get_pair_locks(pair, lock_time + timedelta(minutes=-2))
-    assert len(locks) == 1
-    assert 'PairLock' in str(locks[0])