From e602ac3406ae2d23210811bc2b21503b39303fa9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Oct 2020 10:54:30 +0100 Subject: [PATCH] Introduce Pairlocks middleware --- freqtrade/freqtradebot.py | 32 +++---- freqtrade/persistence/__init__.py | 4 +- freqtrade/persistence/models.py | 50 +--------- freqtrade/persistence/pairlock_middleware.py | 97 ++++++++++++++++++++ freqtrade/rpc/rpc.py | 4 +- freqtrade/strategy/interface.py | 10 +- tests/pairlist/test_pairlocks.py | 81 ++++++++++++++++ tests/rpc/test_rpc_apiserver.py | 6 +- tests/rpc/test_rpc_telegram.py | 6 +- tests/strategy/test_interface.py | 4 +- tests/test_freqtradebot.py | 3 +- tests/test_persistence.py | 49 +--------- 12 files changed, 216 insertions(+), 130 deletions(-) create mode 100644 freqtrade/persistence/pairlock_middleware.py create mode 100644 tests/pairlist/test_pairlocks.py diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6112a599e..5a399801a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -345,23 +345,23 @@ class FreqtradeBot: whitelist = copy.deepcopy(self.active_pair_whitelist) if not whitelist: logger.info("Active pair whitelist is empty.") - else: - # Remove pairs for currently opened trades from the whitelist - for trade in Trade.get_open_trades(): - if trade.pair in whitelist: - whitelist.remove(trade.pair) - logger.debug('Ignoring %s in pair whitelist', trade.pair) + return trades_created + # Remove pairs for currently opened trades from the whitelist + for trade in Trade.get_open_trades(): + if trade.pair in whitelist: + whitelist.remove(trade.pair) + logger.debug('Ignoring %s in pair whitelist', trade.pair) - if not whitelist: - logger.info("No currency pair in active pair whitelist, " - "but checking to sell open trades.") - else: - # Create entity and execute trade for each pair from whitelist - for pair in whitelist: - try: - trades_created += self.create_trade(pair) - except DependencyException as exception: - logger.warning('Unable to create trade for %s: %s', pair, exception) + if not whitelist: + logger.info("No currency pair in active pair whitelist, " + "but checking to sell open trades.") + return trades_created + # Create entity and execute trade for each pair from whitelist + for pair in whitelist: + try: + trades_created += self.create_trade(pair) + except DependencyException as exception: + logger.warning('Unable to create trade for %s: %s', pair, exception) if not trades_created: logger.debug("Found no buy signals for whitelisted currencies. " diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index e184e7d9a..35f2bc406 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,4 +1,4 @@ # flake8: noqa: F401 -from freqtrade.persistence.models import (Order, PairLock, Trade, clean_dry_run_db, cleanup_db, - init_db) +from freqtrade.persistence.models import Order, Trade, clean_dry_run_db, cleanup_db, init_db +from freqtrade.persistence.pairlock_middleware import PairLocks diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 22efed78d..62b033bdf 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -684,19 +684,7 @@ class PairLock(_DECL_BASE): f'lock_end_time={lock_end_time})') @staticmethod - def lock_pair(pair: str, until: datetime, reason: str = None) -> None: - lock = PairLock( - pair=pair, - lock_time=datetime.now(timezone.utc), - lock_end_time=until, - reason=reason, - active=True - ) - PairLock.session.add(lock) - PairLock.session.flush() - - @staticmethod - def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List['PairLock']: + def query_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> Query: """ Get all locks for this pair :param pair: Pair to check for. Returns all current locks if pair is empty @@ -713,41 +701,7 @@ class PairLock(_DECL_BASE): filters.append(PairLock.pair == pair) return PairLock.query.filter( *filters - ).all() - - @staticmethod - def unlock_pair(pair: str, now: Optional[datetime] = None) -> None: - """ - Release all locks for this pair. - :param pair: Pair to unlock - :param now: Datetime object (generated via datetime.now(timezone.utc)). - defaults to datetime.utcnow() - """ - if not now: - now = datetime.now(timezone.utc) - - logger.info(f"Releasing all locks for {pair}.") - locks = PairLock.get_pair_locks(pair, now) - for lock in locks: - lock.active = False - PairLock.session.flush() - - @staticmethod - def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool: - """ - :param pair: Pair to check for - :param now: Datetime object (generated via datetime.now(timezone.utc)). - defaults to datetime.utcnow() - """ - if not now: - now = datetime.now(timezone.utc) - - return PairLock.query.filter( - PairLock.pair == pair, - func.datetime(PairLock.lock_end_time) >= now, - # Only active locks - PairLock.active.is_(True), - ).first() is not None + ) def to_json(self) -> Dict[str, Any]: return { diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py new file mode 100644 index 000000000..ca2c31e36 --- /dev/null +++ b/freqtrade/persistence/pairlock_middleware.py @@ -0,0 +1,97 @@ + + +import logging +from datetime import datetime, timezone +from typing import List, Optional + +from freqtrade.persistence.models import PairLock + + +logger = logging.getLogger(__name__) + + +class PairLocks(): + """ + Pairlocks intermediate class + + """ + + use_db = True + locks: List[PairLock] = [] + + @staticmethod + def lock_pair(pair: str, until: datetime, reason: str = None) -> None: + lock = PairLock( + pair=pair, + lock_time=datetime.now(timezone.utc), + lock_end_time=until, + reason=reason, + active=True + ) + if PairLocks.use_db: + PairLock.session.add(lock) + PairLock.session.flush() + else: + PairLocks.locks.append(lock) + + @staticmethod + def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: + """ + Get all currently active locks for this pair + :param pair: Pair to check for. Returns all current locks if pair is empty + :param now: Datetime object (generated via datetime.now(timezone.utc)). + defaults to datetime.utcnow() + """ + if not now: + now = datetime.now(timezone.utc) + + if PairLocks.use_db: + return PairLock.query_pair_locks(pair, now).all() + else: + locks = [lock for lock in PairLocks.locks if ( + lock.lock_end_time > now + and lock.active is True + and (pair is None or lock.pair == pair) + )] + return locks + + @staticmethod + def unlock_pair(pair: str, now: Optional[datetime] = None) -> None: + """ + Release all locks for this pair. + :param pair: Pair to unlock + :param now: Datetime object (generated via datetime.now(timezone.utc)). + defaults to datetime.now(timezone.utc) + """ + if not now: + now = datetime.now(timezone.utc) + + logger.info(f"Releasing all locks for {pair}.") + locks = PairLocks.get_pair_locks(pair, now) + for lock in locks: + lock.active = False + if PairLocks.use_db: + PairLock.session.flush() + + @staticmethod + def is_global_lock(now: Optional[datetime] = None) -> bool: + """ + :param now: Datetime object (generated via datetime.now(timezone.utc)). + defaults to datetime.now(timezone.utc) + """ + if not now: + now = datetime.now(timezone.utc) + + return len(PairLocks.get_pair_locks('*', now)) > 0 + + @staticmethod + def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool: + """ + :param pair: Pair to check for + :param now: Datetime object (generated via datetime.now(timezone.utc)). + defaults to datetime.now(timezone.utc) + """ + if not now: + now = datetime.now(timezone.utc) + + return len(PairLocks.get_pair_locks(pair, now)) > 0 or PairLocks.is_global_lock(now) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index de8bcaefb..10aaf56fa 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -19,7 +19,7 @@ from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.loggers import bufferHandler from freqtrade.misc import shorten_date -from freqtrade.persistence import PairLock, Trade +from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State from freqtrade.strategy.interface import SellType @@ -604,7 +604,7 @@ class RPC: if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') - locks = PairLock.get_pair_locks(None) + locks = PairLocks.get_pair_locks(None) return { 'lock_count': len(locks), 'locks': [lock.to_json() for lock in locks] diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e6256cafb..1c6aa535d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -17,7 +17,7 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes from freqtrade.exchange.exchange import timeframe_to_next_date -from freqtrade.persistence import PairLock, Trade +from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -288,7 +288,7 @@ class IStrategy(ABC): Needs to be timezone aware `datetime.now(timezone.utc)` :param reason: Optional string explaining why the pair was locked. """ - PairLock.lock_pair(pair, until, reason) + PairLocks.lock_pair(pair, until, reason) def unlock_pair(self, pair: str) -> None: """ @@ -297,7 +297,7 @@ class IStrategy(ABC): manually from within the strategy, to allow an easy way to unlock pairs. :param pair: Unlock pair to allow trading again """ - PairLock.unlock_pair(pair, datetime.now(timezone.utc)) + PairLocks.unlock_pair(pair, datetime.now(timezone.utc)) def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool: """ @@ -312,10 +312,10 @@ class IStrategy(ABC): if not candle_date: # Simple call ... - return PairLock.is_pair_locked(pair, candle_date) + return PairLocks.is_pair_locked(pair, candle_date) else: lock_time = timeframe_to_next_date(self.timeframe, candle_date) - return PairLock.is_pair_locked(pair, lock_time) + return PairLocks.is_pair_locked(pair, lock_time) def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ diff --git a/tests/pairlist/test_pairlocks.py b/tests/pairlist/test_pairlocks.py new file mode 100644 index 000000000..3ed7d643e --- /dev/null +++ b/tests/pairlist/test_pairlocks.py @@ -0,0 +1,81 @@ +from datetime import datetime, timedelta, timezone + +import arrow +import pytest + +from freqtrade.persistence import PairLocks +from freqtrade.persistence.models import PairLock + + +@pytest.mark.parametrize('use_db', (False, True)) +@pytest.mark.usefixtures("init_persistence") +def test_PairLocks(use_db): + # No lock should be present + if use_db: + assert len(PairLock.query.all()) == 0 + else: + PairLocks.use_db = False + + assert PairLocks.use_db == use_db + + pair = 'ETH/BTC' + assert not PairLocks.is_pair_locked(pair) + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + # ETH/BTC locked for 4 minutes + assert PairLocks.is_pair_locked(pair) + + # XRP/BTC should not be locked now + pair = 'XRP/BTC' + assert not PairLocks.is_pair_locked(pair) + # Unlocking a pair that's not locked should not raise an error + PairLocks.unlock_pair(pair) + + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + assert PairLocks.is_pair_locked(pair) + + # Get both locks from above + locks = PairLocks.get_pair_locks(None) + assert len(locks) == 2 + + # Unlock original pair + pair = 'ETH/BTC' + PairLocks.unlock_pair(pair) + assert not PairLocks.is_pair_locked(pair) + assert not PairLocks.is_global_lock() + + pair = 'BTC/USDT' + # Lock until 14:30 + lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc) + PairLocks.lock_pair(pair, lock_time) + + assert not PairLocks.is_pair_locked(pair) + assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-10)) + assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-10)) + assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) + assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50)) + + # Should not be locked after time expired + assert not PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=10)) + + locks = PairLocks.get_pair_locks(pair, lock_time + timedelta(minutes=-2)) + assert len(locks) == 1 + assert 'PairLock' in str(locks[0]) + + # Unlock all + PairLocks.unlock_pair(pair, lock_time + timedelta(minutes=-2)) + assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50)) + + # Global lock + PairLocks.lock_pair('*', lock_time) + assert PairLocks.is_global_lock(lock_time + timedelta(minutes=-50)) + # Global lock also locks every pair seperately + assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) + assert PairLocks.is_pair_locked('XRP/USDT', lock_time + timedelta(minutes=-50)) + + if use_db: + assert len(PairLock.query.all()) > 0 + else: + # Nothing was pushed to the database + assert len(PairLock.query.all()) == 0 + # Reset use-db variable + PairLocks.use_db = True diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 34e959875..0dd15a777 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -12,7 +12,7 @@ from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ from freqtrade.loggers import setup_logging, setup_logging_pre -from freqtrade.persistence import PairLock, Trade +from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc.api_server import BASE_URI, ApiServer from freqtrade.state import State from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, patch_get_signal @@ -339,8 +339,8 @@ def test_api_locks(botclient): assert rc.json['lock_count'] == 0 assert rc.json['lock_count'] == len(rc.json['locks']) - PairLock.lock_pair('ETH/BTC', datetime.utcnow() + timedelta(minutes=4), 'randreason') - PairLock.lock_pair('XRP/BTC', datetime.utcnow() + timedelta(minutes=20), 'deadbeef') + PairLocks.lock_pair('ETH/BTC', datetime.utcnow() + timedelta(minutes=4), 'randreason') + PairLocks.lock_pair('XRP/BTC', datetime.utcnow() + timedelta(minutes=20), 'deadbeef') rc = client_get(client, f"{BASE_URI}/locks") assert_response(rc) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index c412313ad..f1246005f 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -18,7 +18,7 @@ from freqtrade.constants import CANCEL_REASON from freqtrade.edge import PairInfo from freqtrade.freqtradebot import FreqtradeBot from freqtrade.loggers import setup_logging -from freqtrade.persistence import PairLock, Trade +from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPCMessageType from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State @@ -1047,8 +1047,8 @@ def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) -> None msg_mock.reset_mock() freqtradebot.state = State.RUNNING - PairLock.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason') - PairLock.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef') + PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason') + PairLocks.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef') telegram._locks(update=update, context=MagicMock()) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index dc5cd47e7..96d4882da 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -11,7 +11,7 @@ from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data from freqtrade.exceptions import StrategyError -from freqtrade.persistence import PairLock, Trade +from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from tests.conftest import log_has, log_has_re @@ -364,7 +364,7 @@ def test_is_pair_locked(default_conf): default_conf.update({'strategy': 'DefaultStrategy'}) strategy = StrategyResolver.load_strategy(default_conf) # No lock should be present - assert len(PairLock.query.all()) == 0 + assert len(PairLocks.get_pair_locks(None)) == 0 pair = 'ETH/BTC' assert not strategy.is_pair_locked(pair) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2a1b0c3cc..29df9c012 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -15,7 +15,8 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie InvalidOrderException, OperationalException, PricingError, TemporaryError) from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.persistence import Order, PairLock, Trade +from freqtrade.persistence import Order, PairLocks, Trade +from freqtrade.persistence.models import PairLock from freqtrade.rpc import RPCMessageType from freqtrade.state import RunMode, State from freqtrade.strategy.interface import SellCheckTuple, SellType diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 243da3396..4216565ac 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1,6 +1,5 @@ # pragma pylint: disable=missing-docstring, C0103 import logging -from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock import arrow @@ -9,7 +8,7 @@ from sqlalchemy import create_engine from freqtrade import constants from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.persistence import Order, PairLock, Trade, clean_dry_run_db, init_db +from freqtrade.persistence import Order, Trade, clean_dry_run_db, init_db from tests.conftest import create_mock_trades, log_has, log_has_re @@ -1159,49 +1158,3 @@ def test_select_order(fee): assert order.ft_order_side == 'stoploss' order = trades[4].select_order('sell', False) assert order is None - - -@pytest.mark.usefixtures("init_persistence") -def test_PairLock(default_conf): - # No lock should be present - assert len(PairLock.query.all()) == 0 - - pair = 'ETH/BTC' - assert not PairLock.is_pair_locked(pair) - PairLock.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) - # ETH/BTC locked for 4 minutes - assert PairLock.is_pair_locked(pair) - - # XRP/BTC should not be locked now - pair = 'XRP/BTC' - assert not PairLock.is_pair_locked(pair) - # Unlocking a pair that's not locked should not raise an error - PairLock.unlock_pair(pair) - - PairLock.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) - assert PairLock.is_pair_locked(pair) - - # Get both locks from above - locks = PairLock.get_pair_locks(None) - assert len(locks) == 2 - - # Unlock original pair - pair = 'ETH/BTC' - PairLock.unlock_pair(pair) - assert not PairLock.is_pair_locked(pair) - - pair = 'BTC/USDT' - # Lock until 14:30 - lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc) - PairLock.lock_pair(pair, lock_time) - - assert not PairLock.is_pair_locked(pair) - assert PairLock.is_pair_locked(pair, lock_time + timedelta(minutes=-10)) - assert PairLock.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) - - # Should not be locked after time expired - assert not PairLock.is_pair_locked(pair, lock_time + timedelta(minutes=10)) - - locks = PairLock.get_pair_locks(pair, lock_time + timedelta(minutes=-2)) - assert len(locks) == 1 - assert 'PairLock' in str(locks[0])