Merge pull request #3895 from freqtrade/pairlock/middleware

Pairlock middleware
This commit is contained in:
Matthias 2020-10-27 19:44:13 +01:00 committed by GitHub
commit 58a92dc3da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 239 additions and 147 deletions

View File

@ -704,7 +704,7 @@ To verify if a pair is currently locked, use `self.is_pair_locked(pair)`.
Locked pairs will always be rounded up to the next candle. So assuming a `5m` timeframe, a lock with `until` set to 10:18 will lock the pair until the candle from 10:15-10:20 will be finished.
!!! Warning
Locking pairs is not functioning during backtesting.
Locking pairs is not available during backtesting.
#### Pair locking example

View File

@ -4,7 +4,7 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade()
import copy
import logging
import traceback
from datetime import datetime
from datetime import datetime, timezone
from math import isclose
from threading import Lock
from typing import Any, Dict, List, Optional
@ -19,10 +19,10 @@ from freqtrade.data.dataprovider import DataProvider
from freqtrade.edge import Edge
from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
InvalidOrderException, PricingError)
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import safe_value_fallback, safe_value_fallback2
from freqtrade.pairlist.pairlistmanager import PairListManager
from freqtrade.persistence import Order, Trade, cleanup_db, init_db
from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.rpc import RPCManager, RPCMessageType
from freqtrade.state import State
@ -72,6 +72,8 @@ class FreqtradeBot:
self.wallets = Wallets(self.config, self.exchange)
PairLocks.timeframe = self.config['timeframe']
self.pairlists = PairListManager(self.exchange, self.config)
self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists)
@ -345,27 +347,27 @@ class FreqtradeBot:
whitelist = copy.deepcopy(self.active_pair_whitelist)
if not whitelist:
logger.info("Active pair whitelist is empty.")
else:
# Remove pairs for currently opened trades from the whitelist
for trade in Trade.get_open_trades():
if trade.pair in whitelist:
whitelist.remove(trade.pair)
logger.debug('Ignoring %s in pair whitelist', trade.pair)
return trades_created
# Remove pairs for currently opened trades from the whitelist
for trade in Trade.get_open_trades():
if trade.pair in whitelist:
whitelist.remove(trade.pair)
logger.debug('Ignoring %s in pair whitelist', trade.pair)
if not whitelist:
logger.info("No currency pair in active pair whitelist, "
"but checking to sell open trades.")
else:
# Create entity and execute trade for each pair from whitelist
for pair in whitelist:
try:
trades_created += self.create_trade(pair)
except DependencyException as exception:
logger.warning('Unable to create trade for %s: %s', pair, exception)
if not whitelist:
logger.info("No currency pair in active pair whitelist, "
"but checking to sell open trades.")
return trades_created
# Create entity and execute trade for each pair from whitelist
for pair in whitelist:
try:
trades_created += self.create_trade(pair)
except DependencyException as exception:
logger.warning('Unable to create trade for %s: %s', pair, exception)
if not trades_created:
logger.debug("Found no buy signals for whitelisted currencies. "
"Trying again...")
if not trades_created:
logger.debug("Found no buy signals for whitelisted currencies. "
"Trying again...")
return trades_created
@ -937,7 +939,7 @@ class FreqtradeBot:
self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
stoploss_order=True)
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['timeframe']),
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
self._notify_sell(trade, "stoploss")
return True
@ -1264,7 +1266,7 @@ class FreqtradeBot:
Trade.session.flush()
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['timeframe']),
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
self._notify_sell(trade, order_type)

View File

@ -1,4 +1,4 @@
# flake8: noqa: F401
from freqtrade.persistence.models import (Order, PairLock, Trade, clean_dry_run_db, cleanup_db,
init_db)
from freqtrade.persistence.models import Order, Trade, clean_dry_run_db, cleanup_db, init_db
from freqtrade.persistence.pairlock_middleware import PairLocks

View File

@ -684,70 +684,21 @@ class PairLock(_DECL_BASE):
f'lock_end_time={lock_end_time})')
@staticmethod
def lock_pair(pair: str, until: datetime, reason: str = None) -> None:
lock = PairLock(
pair=pair,
lock_time=datetime.now(timezone.utc),
lock_end_time=until,
reason=reason,
active=True
)
PairLock.session.add(lock)
PairLock.session.flush()
@staticmethod
def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List['PairLock']:
def query_pair_locks(pair: Optional[str], now: datetime) -> Query:
"""
Get all locks for this pair
:param pair: Pair to check for. Returns all current locks if pair is empty
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.utcnow()
"""
if not now:
now = datetime.now(timezone.utc)
filters = [func.datetime(PairLock.lock_end_time) >= now,
filters = [PairLock.lock_end_time > now,
# Only active locks
PairLock.active.is_(True), ]
if pair:
filters.append(PairLock.pair == pair)
return PairLock.query.filter(
*filters
).all()
@staticmethod
def unlock_pair(pair: str, now: Optional[datetime] = None) -> None:
"""
Release all locks for this pair.
:param pair: Pair to unlock
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.utcnow()
"""
if not now:
now = datetime.now(timezone.utc)
logger.info(f"Releasing all locks for {pair}.")
locks = PairLock.get_pair_locks(pair, now)
for lock in locks:
lock.active = False
PairLock.session.flush()
@staticmethod
def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool:
"""
:param pair: Pair to check for
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.utcnow()
"""
if not now:
now = datetime.now(timezone.utc)
return PairLock.query.filter(
PairLock.pair == pair,
func.datetime(PairLock.lock_end_time) >= now,
# Only active locks
PairLock.active.is_(True),
).first() is not None
)
def to_json(self) -> Dict[str, Any]:
return {

View File

@ -0,0 +1,99 @@
import logging
from datetime import datetime, timezone
from typing import List, Optional
from freqtrade.exchange import timeframe_to_next_date
from freqtrade.persistence.models import PairLock
logger = logging.getLogger(__name__)
class PairLocks():
"""
Pairlocks middleware class
Abstracts the database layer away so it becomes optional - which will be necessary to support
backtesting and hyperopt in the future.
"""
use_db = True
locks: List[PairLock] = []
timeframe: str = ''
@staticmethod
def lock_pair(pair: str, until: datetime, reason: str = None) -> None:
lock = PairLock(
pair=pair,
lock_time=datetime.now(timezone.utc),
lock_end_time=timeframe_to_next_date(PairLocks.timeframe, until),
reason=reason,
active=True
)
if PairLocks.use_db:
PairLock.session.add(lock)
PairLock.session.flush()
else:
PairLocks.locks.append(lock)
@staticmethod
def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]:
"""
Get all currently active locks for this pair
:param pair: Pair to check for. Returns all current locks if pair is empty
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.now(timezone.utc)
"""
if not now:
now = datetime.now(timezone.utc)
if PairLocks.use_db:
return PairLock.query_pair_locks(pair, now).all()
else:
locks = [lock for lock in PairLocks.locks if (
lock.lock_end_time >= now
and lock.active is True
and (pair is None or lock.pair == pair)
)]
return locks
@staticmethod
def unlock_pair(pair: str, now: Optional[datetime] = None) -> None:
"""
Release all locks for this pair.
:param pair: Pair to unlock
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.now(timezone.utc)
"""
if not now:
now = datetime.now(timezone.utc)
logger.info(f"Releasing all locks for {pair}.")
locks = PairLocks.get_pair_locks(pair, now)
for lock in locks:
lock.active = False
if PairLocks.use_db:
PairLock.session.flush()
@staticmethod
def is_global_lock(now: Optional[datetime] = None) -> bool:
"""
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.now(timezone.utc)
"""
if not now:
now = datetime.now(timezone.utc)
return len(PairLocks.get_pair_locks('*', now)) > 0
@staticmethod
def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool:
"""
:param pair: Pair to check for
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.now(timezone.utc)
"""
if not now:
now = datetime.now(timezone.utc)
return len(PairLocks.get_pair_locks(pair, now)) > 0 or PairLocks.is_global_lock(now)

View File

@ -19,7 +19,7 @@ from freqtrade.exceptions import ExchangeError, PricingError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
from freqtrade.loggers import bufferHandler
from freqtrade.misc import shorten_date
from freqtrade.persistence import PairLock, Trade
from freqtrade.persistence import PairLocks, Trade
from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
from freqtrade.state import State
from freqtrade.strategy.interface import SellType
@ -604,7 +604,7 @@ class RPC:
if self._freqtrade.state != State.RUNNING:
raise RPCException('trader is not running')
locks = PairLock.get_pair_locks(None)
locks = PairLocks.get_pair_locks(None)
return {
'lock_count': len(locks),
'locks': [lock.to_json() for lock in locks]

View File

@ -17,7 +17,7 @@ from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import OperationalException, StrategyError
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.exchange.exchange import timeframe_to_next_date
from freqtrade.persistence import PairLock, Trade
from freqtrade.persistence import PairLocks, Trade
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from freqtrade.wallets import Wallets
@ -288,7 +288,7 @@ class IStrategy(ABC):
Needs to be timezone aware `datetime.now(timezone.utc)`
:param reason: Optional string explaining why the pair was locked.
"""
PairLock.lock_pair(pair, until, reason)
PairLocks.lock_pair(pair, until, reason)
def unlock_pair(self, pair: str) -> None:
"""
@ -297,7 +297,7 @@ class IStrategy(ABC):
manually from within the strategy, to allow an easy way to unlock pairs.
:param pair: Unlock pair to allow trading again
"""
PairLock.unlock_pair(pair, datetime.now(timezone.utc))
PairLocks.unlock_pair(pair, datetime.now(timezone.utc))
def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool:
"""
@ -312,10 +312,10 @@ class IStrategy(ABC):
if not candle_date:
# Simple call ...
return PairLock.is_pair_locked(pair, candle_date)
return PairLocks.is_pair_locked(pair, candle_date)
else:
lock_time = timeframe_to_next_date(self.timeframe, candle_date)
return PairLock.is_pair_locked(pair, lock_time)
return PairLocks.is_pair_locked(pair, lock_time)
def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""

View File

@ -1,6 +1,6 @@
import copy
import logging
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from random import randint
from unittest.mock import MagicMock, Mock, PropertyMock, patch
@ -2300,6 +2300,9 @@ def test_timeframe_to_next_date():
date = datetime.now(tz=timezone.utc)
assert timeframe_to_next_date("5m") > date
date = datetime(2019, 8, 12, 13, 30, 0, tzinfo=timezone.utc)
assert timeframe_to_next_date("5m", date) == date + timedelta(minutes=5)
@pytest.mark.parametrize("market_symbol,base,quote,exchange,add_dict,expected_result", [
("BTC/USDT", 'BTC', 'USDT', "binance", {}, True),

View File

@ -0,0 +1,81 @@
from datetime import datetime, timedelta, timezone
import arrow
import pytest
from freqtrade.persistence import PairLocks
from freqtrade.persistence.models import PairLock
@pytest.mark.parametrize('use_db', (False, True))
@pytest.mark.usefixtures("init_persistence")
def test_PairLocks(use_db):
# No lock should be present
if use_db:
assert len(PairLock.query.all()) == 0
else:
PairLocks.use_db = False
assert PairLocks.use_db == use_db
pair = 'ETH/BTC'
assert not PairLocks.is_pair_locked(pair)
PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
# ETH/BTC locked for 4 minutes
assert PairLocks.is_pair_locked(pair)
# XRP/BTC should not be locked now
pair = 'XRP/BTC'
assert not PairLocks.is_pair_locked(pair)
# Unlocking a pair that's not locked should not raise an error
PairLocks.unlock_pair(pair)
PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
assert PairLocks.is_pair_locked(pair)
# Get both locks from above
locks = PairLocks.get_pair_locks(None)
assert len(locks) == 2
# Unlock original pair
pair = 'ETH/BTC'
PairLocks.unlock_pair(pair)
assert not PairLocks.is_pair_locked(pair)
assert not PairLocks.is_global_lock()
pair = 'BTC/USDT'
# Lock until 14:30
lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc)
PairLocks.lock_pair(pair, lock_time)
assert not PairLocks.is_pair_locked(pair)
assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-10))
assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-10))
assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50))
# Should not be locked after time expired
assert not PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=10))
locks = PairLocks.get_pair_locks(pair, lock_time + timedelta(minutes=-2))
assert len(locks) == 1
assert 'PairLock' in str(locks[0])
# Unlock all
PairLocks.unlock_pair(pair, lock_time + timedelta(minutes=-2))
assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50))
# Global lock
PairLocks.lock_pair('*', lock_time)
assert PairLocks.is_global_lock(lock_time + timedelta(minutes=-50))
# Global lock also locks every pair seperately
assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
assert PairLocks.is_pair_locked('XRP/USDT', lock_time + timedelta(minutes=-50))
if use_db:
assert len(PairLock.query.all()) > 0
else:
# Nothing was pushed to the database
assert len(PairLock.query.all()) == 0
# Reset use-db variable
PairLocks.use_db = True

View File

@ -2,7 +2,7 @@
Unit test file for rpc/api_server.py
"""
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest.mock import ANY, MagicMock, PropertyMock
@ -12,7 +12,7 @@ from requests.auth import _basic_auth_str
from freqtrade.__init__ import __version__
from freqtrade.loggers import setup_logging, setup_logging_pre
from freqtrade.persistence import PairLock, Trade
from freqtrade.persistence import PairLocks, Trade
from freqtrade.rpc.api_server import BASE_URI, ApiServer
from freqtrade.state import State
from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, patch_get_signal
@ -339,8 +339,8 @@ def test_api_locks(botclient):
assert rc.json['lock_count'] == 0
assert rc.json['lock_count'] == len(rc.json['locks'])
PairLock.lock_pair('ETH/BTC', datetime.utcnow() + timedelta(minutes=4), 'randreason')
PairLock.lock_pair('XRP/BTC', datetime.utcnow() + timedelta(minutes=20), 'deadbeef')
PairLocks.lock_pair('ETH/BTC', datetime.now(timezone.utc) + timedelta(minutes=4), 'randreason')
PairLocks.lock_pair('XRP/BTC', datetime.now(timezone.utc) + timedelta(minutes=20), 'deadbeef')
rc = client_get(client, f"{BASE_URI}/locks")
assert_response(rc)

View File

@ -18,7 +18,7 @@ from freqtrade.constants import CANCEL_REASON
from freqtrade.edge import PairInfo
from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.loggers import setup_logging
from freqtrade.persistence import PairLock, Trade
from freqtrade.persistence import PairLocks, Trade
from freqtrade.rpc import RPCMessageType
from freqtrade.rpc.telegram import Telegram, authorized_only
from freqtrade.state import State
@ -1047,8 +1047,8 @@ def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) -> None
msg_mock.reset_mock()
freqtradebot.state = State.RUNNING
PairLock.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason')
PairLock.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef')
PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason')
PairLocks.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef')
telegram._locks(update=update, context=MagicMock())

View File

@ -11,7 +11,7 @@ from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history import load_data
from freqtrade.exceptions import StrategyError
from freqtrade.persistence import PairLock, Trade
from freqtrade.persistence import PairLocks, Trade
from freqtrade.resolvers import StrategyResolver
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from tests.conftest import log_has, log_has_re
@ -362,13 +362,14 @@ def test__analyze_ticker_internal_skip_analyze(ohlcv_history, mocker, caplog) ->
@pytest.mark.usefixtures("init_persistence")
def test_is_pair_locked(default_conf):
default_conf.update({'strategy': 'DefaultStrategy'})
PairLocks.timeframe = default_conf['timeframe']
strategy = StrategyResolver.load_strategy(default_conf)
# No lock should be present
assert len(PairLock.query.all()) == 0
assert len(PairLocks.get_pair_locks(None)) == 0
pair = 'ETH/BTC'
assert not strategy.is_pair_locked(pair)
strategy.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
strategy.lock_pair(pair, arrow.now(timezone.utc).shift(minutes=4).datetime)
# ETH/BTC locked for 4 minutes
assert strategy.is_pair_locked(pair)
@ -387,7 +388,8 @@ def test_is_pair_locked(default_conf):
pair = 'BTC/USDT'
# Lock until 14:30
lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc)
strategy.lock_pair(pair, lock_time)
# Subtract 2 seconds, as locking rounds up to the next candle.
strategy.lock_pair(pair, lock_time - timedelta(seconds=2))
assert not strategy.is_pair_locked(pair)
# latest candle is from 14:20, lock goes to 14:30

View File

@ -15,7 +15,8 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie
InvalidOrderException, OperationalException, PricingError,
TemporaryError)
from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.persistence import Order, PairLock, Trade
from freqtrade.persistence import Order, Trade
from freqtrade.persistence.models import PairLock
from freqtrade.rpc import RPCMessageType
from freqtrade.state import RunMode, State
from freqtrade.strategy.interface import SellCheckTuple, SellType

View File

@ -1,6 +1,5 @@
# pragma pylint: disable=missing-docstring, C0103
import logging
from datetime import datetime, timedelta, timezone
from unittest.mock import MagicMock
import arrow
@ -9,7 +8,7 @@ from sqlalchemy import create_engine
from freqtrade import constants
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.persistence import Order, PairLock, Trade, clean_dry_run_db, init_db
from freqtrade.persistence import Order, Trade, clean_dry_run_db, init_db
from tests.conftest import create_mock_trades, log_has, log_has_re
@ -1159,49 +1158,3 @@ def test_select_order(fee):
assert order.ft_order_side == 'stoploss'
order = trades[4].select_order('sell', False)
assert order is None
@pytest.mark.usefixtures("init_persistence")
def test_PairLock(default_conf):
# No lock should be present
assert len(PairLock.query.all()) == 0
pair = 'ETH/BTC'
assert not PairLock.is_pair_locked(pair)
PairLock.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
# ETH/BTC locked for 4 minutes
assert PairLock.is_pair_locked(pair)
# XRP/BTC should not be locked now
pair = 'XRP/BTC'
assert not PairLock.is_pair_locked(pair)
# Unlocking a pair that's not locked should not raise an error
PairLock.unlock_pair(pair)
PairLock.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
assert PairLock.is_pair_locked(pair)
# Get both locks from above
locks = PairLock.get_pair_locks(None)
assert len(locks) == 2
# Unlock original pair
pair = 'ETH/BTC'
PairLock.unlock_pair(pair)
assert not PairLock.is_pair_locked(pair)
pair = 'BTC/USDT'
# Lock until 14:30
lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc)
PairLock.lock_pair(pair, lock_time)
assert not PairLock.is_pair_locked(pair)
assert PairLock.is_pair_locked(pair, lock_time + timedelta(minutes=-10))
assert PairLock.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
# Should not be locked after time expired
assert not PairLock.is_pair_locked(pair, lock_time + timedelta(minutes=10))
locks = PairLock.get_pair_locks(pair, lock_time + timedelta(minutes=-2))
assert len(locks) == 1
assert 'PairLock' in str(locks[0])