adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1018 from freqtrade/feat/sell_reason

Browse Source 
Record sell reason
This commit is contained in:
Janne Sinivirta 2018-07-24 09:09:45 +03:00 committed by GitHub
commit 0b3190552e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 162 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/backtesting.md
View File

@ -83,7 +83,7 @@ with filename.open() as file:
data = json.load(file)
columns = ["pair", "profit", "opents", "closets", "index", "duration",
"open_rate", "close_rate", "open_at_end"]
"open_rate", "close_rate", "open_at_end", "sell_reason"]
df = pd.DataFrame(data, columns=columns)
df['opents'] = pd.to_datetime(df['opents'],
@ -98,6 +98,8 @@ df['closets'] = pd.to_datetime(df['closets'],
)
```
If you have some ideas for interesting / helpful backtest data analysis, feel free to submit a PR so the community can benefit from it.
#### Exporting trades to file specifying a custom filename
```bash

15
freqtrade/freqtradebot.py
View File

@ -20,6 +20,7 @@ from freqtrade.fiat_convert import CryptoToFiatConverter
from freqtrade.persistence import Trade
from freqtrade.rpc import RPCManager, RPCMessageType
from freqtrade.state import State
from freqtrade.strategy.interface import SellType
from freqtrade.strategy.resolver import IStrategy, StrategyResolver
logger = logging.getLogger(__name__)
@ -53,7 +54,6 @@ class FreqtradeBot(object):
self.rpc: RPCManager = RPCManager(self)
self.persistence = None
self.exchange = Exchange(self.config)
self._init_modules()
def _init_modules(self) -> None:
@ -392,7 +392,9 @@ class FreqtradeBot(object):
open_rate_requested=buy_limit,
open_date=datetime.utcnow(),
exchange=self.exchange.id,
open_order_id=order_id
open_order_id=order_id,
strategy=self.strategy.get_strategy_name(),
ticker_interval=constants.TICKER_INTERVAL_MINUTES[self.config['ticker_interval']]
)
Trade.session.add(trade)
Trade.session.flush()
@ -505,8 +507,9 @@ class FreqtradeBot(object):
(buy, sell) = self.strategy.get_signal(self.exchange,
trade.pair, self.strategy.ticker_interval)
if self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell):
self.execute_sell(trade, current_rate)
should_sell = self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell)
if should_sell.sell_flag:
self.execute_sell(trade, current_rate, should_sell.sell_type)
return True
logger.info('Found no sell signals for whitelisted currencies. Trying again..')
return False
@ -607,17 +610,19 @@ class FreqtradeBot(object):
# TODO: figure out how to handle partially complete sell orders
return False
def execute_sell(self, trade: Trade, limit: float) -> None:
def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> None:
"""
Executes a limit sell for the given trade and limit
:param trade: Trade instance
:param limit: limit rate for the sell order
:param sellreason: Reason the sell was triggered
:return: None
"""
# Execute sell and update trade record
order_id = self.exchange.sell(str(trade.pair), limit, trade.amount)['id']
trade.open_order_id = order_id
trade.close_rate_requested = limit
trade.sell_reason = sell_reason.value
profit_trade = trade.calc_profit(rate=limit)
current_rate = self.exchange.get_ticker(trade.pair)['bid']

49
freqtrade/optimize/backtesting.py
View File

@ -20,6 +20,7 @@ from freqtrade.configuration import Configuration
from freqtrade.exchange import Exchange
from freqtrade.misc import file_dump_json
from freqtrade.persistence import Trade
from freqtrade.strategy.interface import SellType
from freqtrade.strategy.resolver import IStrategy, StrategyResolver
logger = logging.getLogger(__name__)
@ -40,6 +41,7 @@ class BacktestResult(NamedTuple):
open_at_end: bool
open_rate: float
close_rate: float
sell_reason: SellType
class Backtesting(object):
@ -120,11 +122,21 @@ class Backtesting(object):
])
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str:
"""
Generate small table outlining Backtest results
"""
tabular_data = []
headers = ['Sell Reason', 'Count']
for reason, count in results['sell_reason'].value_counts().iteritems():
tabular_data.append([reason.value, count])
return tabulate(tabular_data, headers=headers, tablefmt="pipe")
def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None:
records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
t.open_rate, t.close_rate, t.open_at_end)
t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value)
for index, t in results.iterrows()]
if records:
@ -153,8 +165,9 @@ class Backtesting(object):
trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1
buy_signal = sell_row.buy
if self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal,
sell_row.sell):
sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal,
sell_row.sell)
if sell.sell_flag:
return BacktestResult(pair=pair,
profit_percent=trade.calc_profit_percent(rate=sell_row.open),
@ -167,7 +180,8 @@ class Backtesting(object):
close_index=sell_row.Index,
open_at_end=False,
open_rate=buy_row.open,
close_rate=sell_row.open
close_rate=sell_row.open,
sell_reason=sell.sell_type
)
if partial_ticker:
# no sell condition found - trade stil open at end of backtest period
@ -183,7 +197,8 @@ class Backtesting(object):
close_index=sell_row.Index,
open_at_end=True,
open_rate=buy_row.open,
close_rate=sell_row.open
close_rate=sell_row.open,
sell_reason=SellType.FORCE_SELL
)
logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair,
btr.profit_percent, btr.profit_abs)
@ -318,21 +333,31 @@ class Backtesting(object):
self._store_backtest_result(self.config.get('exportfilename'), results)
logger.info(
'\n================================================= '
'BACKTESTING REPORT'
' ==================================================\n'
'\n' + '=' * 49 +
' BACKTESTING REPORT ' +
'=' * 50 + '\n'
'%s',
self._generate_text_table(
data,
results
)
)
# logger.info(
# results[['sell_reason']].groupby('sell_reason').count()
# )
logger.info(
'\n=============================================== '
'LEFT OPEN TRADES REPORT'
' ===============================================\n'
'%s',
'\n' +
' SELL READON STATS '.center(119, '=') +
'\n%s \n',
self._generate_text_table_sell_reason(data, results)
)
logger.info(
'\n' +
' LEFT OPEN TRADES REPORT '.center(119, '=') +
'\n%s',
self._generate_text_table(
data,
results.loc[results.open_at_end]

12
freqtrade/persistence.py
View File

@ -90,6 +90,9 @@ def check_migrate(engine) -> None:
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
max_rate = get_column_def(cols, 'max_rate', '0.0')
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
ticker_interval = get_column_def(cols, 'ticker_interval', 'null')
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
@ -101,7 +104,8 @@ def check_migrate(engine) -> None:
(id, exchange, pair, is_open, fee_open, fee_close, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, open_date, close_date, open_order_id,
stop_loss, initial_stop_loss, max_rate
stop_loss, initial_stop_loss, max_rate, sell_reason, strategy,
ticker_interval
)
select id, lower(exchange),
case
@ -116,7 +120,8 @@ def check_migrate(engine) -> None:
{close_rate_requested} close_rate_requested, close_profit,
stake_amount, amount, open_date, close_date, open_order_id,
{stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss,
{max_rate} max_rate
{max_rate} max_rate, {sell_reason} sell_reason, {strategy} strategy,
{ticker_interval} ticker_interval
from {table_back_name}
""")
@ -172,6 +177,9 @@ class Trade(_DECL_BASE):
initial_stop_loss = Column(Float, nullable=True, default=0.0)
# absolute value of the highest reached price
max_rate = Column(Float, nullable=True, default=0.0)
sell_reason = Column(String, nullable=True)
strategy = Column(String, nullable=True)
ticker_interval = Column(Integer, nullable=True)
def __repr__(self):
open_since = arrow.get(self.open_date).humanize() if self.is_open else 'closed'

3
freqtrade/rpc/rpc.py
View File

@ -16,6 +16,7 @@ from pandas import DataFrame
from freqtrade.misc import shorten_date
from freqtrade.persistence import Trade
from freqtrade.state import State
from freqtrade.strategy.interface import SellType
logger = logging.getLogger(__name__)
@ -344,7 +345,7 @@ class RPC(object):
# Get current rate and execute sell
current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid']
self._freqtrade.execute_sell(trade, current_rate)
self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL)
# ---- EOF def _exec_forcesell ----
if self._freqtrade.state != State.RUNNING:

57
freqtrade/strategy/interface.py
View File

@ -6,7 +6,7 @@ import logging
from abc import ABC, abstractmethod
from datetime import datetime
from enum import Enum
from typing import Dict, List, Tuple
from typing import Dict, List, NamedTuple, Tuple
import arrow
from pandas import DataFrame
@ -27,6 +27,26 @@ class SignalType(Enum):
SELL = "sell"
class SellType(Enum):
"""
Enum to distinguish between sell reasons
"""
ROI = "roi"
STOP_LOSS = "stop_loss"
TRAILING_STOP_LOSS = "trailing_stop_loss"
SELL_SIGNAL = "sell_signal"
FORCE_SELL = "force_sell"
NONE = ""
class SellCheckTuple(NamedTuple):
"""
NamedTuple for Sell type + reason
"""
sell_flag: bool
sell_type: SellType
class IStrategy(ABC):
"""
Interface for freqtrade strategies
@ -69,6 +89,12 @@ class IStrategy(ABC):
:return: DataFrame with sell column
"""
def get_strategy_name(self) -> str:
"""
Returns strategy class name
"""
return self.__class__.__name__
def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame:
"""
Parses the given ticker history and returns a populated DataFrame
@ -137,40 +163,42 @@ class IStrategy(ABC):
)
return buy, sell
def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool:
def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool,
sell: bool) -> SellCheckTuple:
"""
This function evaluate if on the condition required to trigger a sell has been reached
if the threshold is reached and updates the trade record.
:return: True if trade should be sold, False otherwise
"""
current_profit = trade.calc_profit_percent(rate)
if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date,
current_profit=current_profit):
return True
stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date,
current_profit=current_profit)
if stoplossflag.sell_flag:
return stoplossflag
experimental = self.config.get('experimental', {})
if buy and experimental.get('ignore_roi_if_buy_signal', False):
logger.debug('Buy signal still active - not selling.')
return False
return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
# Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee)
if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date):
logger.debug('Required profit reached. Selling..')
return True
return SellCheckTuple(sell_flag=True, sell_type=SellType.ROI)
if experimental.get('sell_profit_only', False):
logger.debug('Checking if trade is profitable..')
if trade.calc_profit(rate=rate) <= 0:
return False
return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
if sell and not buy and experimental.get('use_sell_signal', False):
logger.debug('Sell signal received. Selling..')
return True
return SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL)
return False
return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime,
current_profit: float) -> bool:
current_profit: float) -> SellCheckTuple:
"""
Based on current profit of the trade and configured (trailing) stoploss,
decides to sell or not
@ -182,8 +210,9 @@ class IStrategy(ABC):
# evaluate if the stoploss was hit
if self.stoploss is not None and trade.stop_loss >= current_rate:
selltype = SellType.STOP_LOSS
if trailing_stop:
selltype = SellType.TRAILING_STOP_LOSS
logger.debug(
f"HIT STOP: current price at {current_rate:.6f}, "
f"stop loss is {trade.stop_loss:.6f}, "
@ -192,7 +221,7 @@ class IStrategy(ABC):
logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}")
logger.debug('Stop loss hit.')
return True
return SellCheckTuple(sell_flag=True, sell_type=selltype)
# update the stop loss afterwards, after all by definition it's supposed to be hanging
if trailing_stop:
@ -209,7 +238,7 @@ class IStrategy(ABC):
trade.adjust_stop_loss(current_rate, stop_loss_value)
return False
return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool:
"""

41
freqtrade/tests/optimize/test_backtesting.py
View File

@ -17,6 +17,7 @@ from freqtrade.arguments import Arguments, TimeRange
from freqtrade.optimize.backtesting import (Backtesting, setup_configuration,
start)
from freqtrade.tests.conftest import log_has, patch_exchange
from freqtrade.strategy.interface import SellType
from freqtrade.strategy.default_strategy import DefaultStrategy
@ -406,6 +407,35 @@ def test_generate_text_table(default_conf, mocker):
assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str
def test_generate_text_table_sell_reason(default_conf, mocker):
"""
Test Backtesting.generate_text_table_sell_reason() method
"""
patch_exchange(mocker)
backtesting = Backtesting(default_conf)
results = pd.DataFrame(
{
'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
'profit_percent': [0.1, 0.2, 0.3],
'profit_abs': [0.2, 0.4, 0.5],
'trade_duration': [10, 30, 10],
'profit': [2, 0, 0],
'loss': [0, 0, 1],
'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
}
)
result_str = (
'| Sell Reason | Count |\n'
'|:--------------|--------:|\n'
'| roi | 2 |\n'
'| stop_loss | 1 |'
)
assert backtesting._generate_text_table_sell_reason(
data={'ETH/BTC': {}}, results=results) == result_str
def test_backtesting_start(default_conf, mocker, caplog) -> None:
"""
Test Backtesting.start() method
@ -514,7 +544,9 @@ def test_backtest(default_conf, fee, mocker) -> None:
'trade_duration': [240, 50],
'open_at_end': [False, False],
'open_rate': [0.104445, 0.10302485],
'close_rate': [0.105, 0.10359999]})
'close_rate': [0.105, 0.10359999],
'sell_reason': [SellType.ROI, SellType.ROI]
})
pd.testing.assert_frame_equal(results, expected)
data_pair = data_processed[pair]
for _, t in results.iterrows():
@ -660,7 +692,9 @@ def test_backtest_record(default_conf, fee, mocker):
"open_index": [1, 119, 153, 185],
"close_index": [118, 151, 184, 199],
"trade_duration": [123, 34, 31, 14],
"open_at_end": [False, False, False, True]
"open_at_end": [False, False, False, True],
"sell_reason": [SellType.ROI, SellType.STOP_LOSS,
SellType.ROI, SellType.FORCE_SELL]
})
backtesting._store_backtest_result("backtest-result.json", results)
assert len(results) == 4
@ -673,7 +707,7 @@ def test_backtest_record(default_conf, fee, mocker):
# Below follows just a typecheck of the schema/type of trade-records
oix = None
for (pair, profit, date_buy, date_sell, buy_index, dur,
openr, closer, open_at_end) in records:
openr, closer, open_at_end, sell_reason) in records:
assert pair == 'UNITTEST/BTC'
assert isinstance(profit, float)
# FIX: buy/sell should be converted to ints
@ -682,6 +716,7 @@ def test_backtest_record(default_conf, fee, mocker):
assert isinstance(openr, float)
assert isinstance(closer, float)
assert isinstance(open_at_end, bool)
assert isinstance(sell_reason, str)
isinstance(buy_index, pd._libs.tslib.Timestamp)
if oix:
assert buy_index > oix

21
freqtrade/tests/test_freqtradebot.py
View File

@ -20,6 +20,7 @@ from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.persistence import Trade
from freqtrade.rpc import RPCMessageType
from freqtrade.state import State
from freqtrade.strategy.interface import SellType, SellCheckTuple
from freqtrade.tests.conftest import log_has, patch_coinmarketcap, patch_exchange
@ -1369,7 +1370,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc
get_ticker=ticker_sell_up
)
freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'])
freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI)
assert rpc_mock.call_count == 2
last_msg = rpc_mock.call_args_list[-1][0][0]
@ -1421,7 +1422,8 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets,
get_ticker=ticker_sell_down
)
freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'])
freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'],
sell_reason=SellType.STOP_LOSS)
assert rpc_mock.call_count == 2
last_msg = rpc_mock.call_args_list[-1][0][0]
@ -1474,7 +1476,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee,
)
freqtrade.config = {}
freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'])
freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI)
assert rpc_mock.call_count == 2
last_msg = rpc_mock.call_args_list[-1][0][0]
@ -1524,7 +1526,8 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee,
)
freqtrade.config = {}
freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'])
freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'],
sell_reason=SellType.STOP_LOSS)
assert rpc_mock.call_count == 2
last_msg = rpc_mock.call_args_list[-1][0][0]
@ -1577,6 +1580,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order,
trade.update(limit_buy_order)
patch_get_signal(freqtrade, value=(False, True))
assert freqtrade.handle_trade(trade) is True
assert trade.sell_reason == SellType.SELL_SIGNAL.value
def test_sell_profit_only_disable_profit(default_conf, limit_buy_order,
@ -1612,6 +1616,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order,
trade.update(limit_buy_order)
patch_get_signal(freqtrade, value=(False, True))
assert freqtrade.handle_trade(trade) is True
assert trade.sell_reason == SellType.SELL_SIGNAL.value
def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None:
@ -1640,7 +1645,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market
freqtrade = FreqtradeBot(conf)
patch_get_signal(freqtrade)
freqtrade.strategy.stop_loss_reached = \
lambda current_rate, trade, current_time, current_profit: False
lambda current_rate, trade, current_time, current_profit: SellCheckTuple(
sell_flag=False, sell_type=SellType.NONE)
freqtrade.create_trade()
trade = Trade.query.first()
@ -1684,6 +1690,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke
trade.update(limit_buy_order)
patch_get_signal(freqtrade, value=(False, True))
assert freqtrade.handle_trade(trade) is True
assert trade.sell_reason == SellType.SELL_SIGNAL.value
def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None:
@ -1724,6 +1731,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m
# Test if buy-signal is absent (should sell due to roi = true)
patch_get_signal(freqtrade, value=(False, True))
assert freqtrade.handle_trade(trade) is True
assert trade.sell_reason == SellType.ROI.value
def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None:
@ -1762,6 +1770,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog,
assert log_has(
f'HIT STOP: current price at 0.000001, stop loss is {trade.stop_loss:.6f}, '
f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples)
assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value
def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets,
@ -1825,6 +1834,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets
f'HIT STOP: current price at {buy_price + 0.000002:.6f}, '
f'stop loss is {trade.stop_loss:.6f}, '
f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples)
assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value
def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order,
@ -1867,6 +1877,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order,
# Test if buy-signal is absent
patch_get_signal(freqtrade, value=(False, True))
assert freqtrade.handle_trade(trade) is True
assert trade.sell_reason == SellType.STOP_LOSS.value
def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, caplog, mocker):

3
freqtrade/tests/test_persistence.py
View File

@ -465,6 +465,9 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
assert trade.max_rate == 0.0
assert trade.stop_loss == 0.0
assert trade.initial_stop_loss == 0.0
assert trade.sell_reason is None
assert trade.strategy is None
assert trade.ticker_interval is None
assert log_has("trying trades_bak1", caplog.record_tuples)
assert log_has("trying trades_bak2", caplog.record_tuples)