Merge pull request #1937 from xmatthias/feat/plot_module

move parts of scripts/plot_dataframe.py to main bot code
This commit is contained in:
Matthias 2019-06-22 13:06:30 +02:00 committed by GitHub
commit d8286d7a98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 677 additions and 344 deletions

View File

@ -56,7 +56,7 @@ class Arguments(object):
# Workaround issue in argparse with action='append' and default value # Workaround issue in argparse with action='append' and default value
# (see https://bugs.python.org/issue16399) # (see https://bugs.python.org/issue16399)
if parsed_arg.config is None and not no_default_config: if not no_default_config and parsed_arg.config is None:
parsed_arg.config = [constants.DEFAULT_CONFIG] parsed_arg.config = [constants.DEFAULT_CONFIG]
return parsed_arg return parsed_arg

View File

@ -103,6 +103,9 @@ class Configuration(object):
# Load Optimize configurations # Load Optimize configurations
config = self._load_optimize_config(config) config = self._load_optimize_config(config)
# Add plotting options if available
config = self._load_plot_config(config)
# Set runmode # Set runmode
if not self.runmode: if not self.runmode:
# Handle real mode, infer dry/live from config # Handle real mode, infer dry/live from config
@ -338,6 +341,26 @@ class Configuration(object):
return config return config
def _load_plot_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""
Extract information for sys.argv Plotting configuration
:return: configuration as dictionary
"""
self._args_to_config(config, argname='pairs',
logstring='Using pairs {}')
self._args_to_config(config, argname='indicators1',
logstring='Using indicators1: {}')
self._args_to_config(config, argname='indicators2',
logstring='Using indicators2: {}')
self._args_to_config(config, argname='plot_limit',
logstring='Limiting plot to: {}')
return config
def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]: def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]:
""" """
Validate the configuration follow the Config Schema Validate the configuration follow the Config Schema

View File

@ -1,12 +1,18 @@
""" """
Helpers when analyzing backtest data Helpers when analyzing backtest data
""" """
import logging
from pathlib import Path from pathlib import Path
import numpy as np import numpy as np
import pandas as pd import pandas as pd
import pytz
from freqtrade import persistence
from freqtrade.misc import json_load from freqtrade.misc import json_load
from freqtrade.persistence import Trade
logger = logging.getLogger(__name__)
# must align with columns in backtest.py # must align with columns in backtest.py
BT_DATA_COLUMNS = ["pair", "profitperc", "open_time", "close_time", "index", "duration", BT_DATA_COLUMNS = ["pair", "profitperc", "open_time", "close_time", "index", "duration",
@ -65,3 +71,48 @@ def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int
df2 = df2.set_index('date') df2 = df2.set_index('date')
df_final = df2.resample(freq)[['pair']].count() df_final = df2.resample(freq)[['pair']].count()
return df_final[df_final['pair'] > max_open_trades] return df_final[df_final['pair'] > max_open_trades]
def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame:
"""
Load trades, either from a DB (using dburl) or via a backtest export file.
:param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite)
:param exportfilename: Path to a file exported from backtesting
:returns: Dataframe containing Trades
"""
timeZone = pytz.UTC
trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS)
if db_url:
persistence.init(db_url, clean_open_orders=False)
columns = ["pair", "profit", "open_time", "close_time",
"open_rate", "close_rate", "duration"]
for x in Trade.query.all():
logger.info("date: {}".format(x.open_date))
trades = pd.DataFrame([(t.pair, t.calc_profit(),
t.open_date.replace(tzinfo=timeZone),
t.close_date.replace(tzinfo=timeZone) if t.close_date else None,
t.open_rate, t.close_rate,
t.close_date.timestamp() - t.open_date.timestamp()
if t.close_date else None)
for t in Trade.query.all()],
columns=columns)
elif exportfilename:
trades = load_backtest_data(Path(exportfilename))
return trades
def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame:
"""
Compare trades and backtested pair DataFrames to get trades performed on backtested period
:return: the DataFrame of a trades of period
"""
trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) &
(trades['close_time'] <= dataframe.iloc[-1]['date'])]
return trades

View File

221
freqtrade/plot/plotting.py Normal file
View File

@ -0,0 +1,221 @@
import logging
from typing import List
import pandas as pd
from pathlib import Path
logger = logging.getLogger(__name__)
try:
from plotly import tools
from plotly.offline import plot
import plotly.graph_objs as go
except ImportError:
logger.exception("Module plotly not found \n Please install using `pip install plotly`")
exit(1)
def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots:
"""
Generator all the indicator selected by the user for a specific row
:param fig: Plot figure to append to
:param row: row number for this plot
:param indicators: List of indicators present in the dataframe
:param data: candlestick DataFrame
"""
for indicator in indicators:
if indicator in data:
# TODO: Figure out why scattergl causes problems
scattergl = go.Scatter(
x=data['date'],
y=data[indicator].values,
mode='lines',
name=indicator
)
fig.append_trace(scattergl, row, 1)
else:
logger.info(
'Indicator "%s" ignored. Reason: This indicator is not found '
'in your strategy.',
indicator
)
return fig
def plot_trades(fig, trades: pd.DataFrame):
"""
Plot trades to "fig"
"""
# Trades can be empty
if trades is not None and len(trades) > 0:
trade_buys = go.Scatter(
x=trades["open_time"],
y=trades["open_rate"],
mode='markers',
name='trade_buy',
marker=dict(
symbol='square-open',
size=11,
line=dict(width=2),
color='green'
)
)
# Create description for sell summarizing the trade
desc = trades.apply(lambda row: f"{round(row['profitperc'], 3)}%, {row['sell_reason']}, "
f"{row['duration']}min",
axis=1)
trade_sells = go.Scatter(
x=trades["close_time"],
y=trades["close_rate"],
text=desc,
mode='markers',
name='trade_sell',
marker=dict(
symbol='square-open',
size=11,
line=dict(width=2),
color='red'
)
)
fig.append_trace(trade_buys, 1, 1)
fig.append_trace(trade_sells, 1, 1)
return fig
def generate_graph(
pair: str,
data: pd.DataFrame,
trades: pd.DataFrame = None,
indicators1: List[str] = [],
indicators2: List[str] = [],
) -> go.Figure:
"""
Generate the graph from the data generated by Backtesting or from DB
Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators
:param pair: Pair to Display on the graph
:param data: OHLCV DataFrame containing indicators and buy/sell signals
:param trades: All trades created
:param indicators1: List containing Main plot indicators
:param indicators2: List containing Sub plot indicators
:return: None
"""
# Define the graph
fig = tools.make_subplots(
rows=3,
cols=1,
shared_xaxes=True,
row_width=[1, 1, 4],
vertical_spacing=0.0001,
)
fig['layout'].update(title=pair)
fig['layout']['yaxis1'].update(title='Price')
fig['layout']['yaxis2'].update(title='Volume')
fig['layout']['yaxis3'].update(title='Other')
fig['layout']['xaxis']['rangeslider'].update(visible=False)
# Common information
candles = go.Candlestick(
x=data.date,
open=data.open,
high=data.high,
low=data.low,
close=data.close,
name='Price'
)
fig.append_trace(candles, 1, 1)
if 'buy' in data.columns:
df_buy = data[data['buy'] == 1]
if len(df_buy) > 0:
buys = go.Scatter(
x=df_buy.date,
y=df_buy.close,
mode='markers',
name='buy',
marker=dict(
symbol='triangle-up-dot',
size=9,
line=dict(width=1),
color='green',
)
)
fig.append_trace(buys, 1, 1)
else:
logger.warning("No buy-signals found.")
if 'sell' in data.columns:
df_sell = data[data['sell'] == 1]
if len(df_sell) > 0:
sells = go.Scatter(
x=df_sell.date,
y=df_sell.close,
mode='markers',
name='sell',
marker=dict(
symbol='triangle-down-dot',
size=9,
line=dict(width=1),
color='red',
)
)
fig.append_trace(sells, 1, 1)
else:
logger.warning("No sell-signals found.")
if 'bb_lowerband' in data and 'bb_upperband' in data:
bb_lower = go.Scattergl(
x=data.date,
y=data.bb_lowerband,
name='BB lower',
line={'color': 'rgba(255,255,255,0)'},
)
bb_upper = go.Scattergl(
x=data.date,
y=data.bb_upperband,
name='BB upper',
fill="tonexty",
fillcolor="rgba(0,176,246,0.2)",
line={'color': 'rgba(255,255,255,0)'},
)
fig.append_trace(bb_lower, 1, 1)
fig.append_trace(bb_upper, 1, 1)
# Add indicators to main plot
fig = generate_row(fig=fig, row=1, indicators=indicators1, data=data)
fig = plot_trades(fig, trades)
# Volume goes to row 2
volume = go.Bar(
x=data['date'],
y=data['volume'],
name='Volume'
)
fig.append_trace(volume, 2, 1)
# Add indicators to seperate row
fig = generate_row(fig=fig, row=3, indicators=indicators2, data=data)
return fig
def generate_plot_file(fig, pair, ticker_interval) -> None:
"""
Generate a plot html file from pre populated fig plotly object
:param fig: Plotly Figure to plot
:param pair: Pair to plot (used as filename and Plot title)
:param ticker_interval: Used as part of the filename
:return: None
"""
logger.info('Generate plot file for %s', pair)
pair_name = pair.replace("/", "_")
file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html'
Path("user_data/plots").mkdir(parents=True, exist_ok=True)
plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)),
auto_open=False)

View File

@ -151,6 +151,11 @@ def patch_coinmarketcap(mocker) -> None:
) )
@pytest.fixture(scope='function')
def init_persistence(default_conf):
persistence.init(default_conf['db_url'], default_conf['dry_run'])
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def default_conf(): def default_conf():
""" Returns validated configuration suitable for most tests """ """ Returns validated configuration suitable for most tests """

View File

@ -1,8 +1,15 @@
import pytest from unittest.mock import MagicMock
from pandas import DataFrame
from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data from arrow import Arrow
from freqtrade.data.history import make_testdata_path import pytest
from pandas import DataFrame, to_datetime
from freqtrade.arguments import TimeRange
from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
extract_trades_of_period,
load_backtest_data, load_trades)
from freqtrade.data.history import load_pair_history, make_testdata_path
from freqtrade.tests.test_persistence import create_mock_trades
def test_load_backtest_data(): def test_load_backtest_data():
@ -19,3 +26,59 @@ def test_load_backtest_data():
with pytest.raises(ValueError, match=r"File .* does not exist\."): with pytest.raises(ValueError, match=r"File .* does not exist\."):
load_backtest_data(str("filename") + "nofile") load_backtest_data(str("filename") + "nofile")
def test_load_trades_file(default_conf, fee, mocker):
# Real testing of load_backtest_data is done in test_load_backtest_data
lbt = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock())
filename = make_testdata_path(None) / "backtest-result_test.json"
load_trades(db_url=None, exportfilename=filename)
assert lbt.call_count == 1
@pytest.mark.usefixtures("init_persistence")
def test_load_trades_db(default_conf, fee, mocker):
create_mock_trades(fee)
# remove init so it does not init again
init_mock = mocker.patch('freqtrade.persistence.init', MagicMock())
trades = load_trades(db_url=default_conf['db_url'], exportfilename=None)
assert init_mock.call_count == 1
assert len(trades) == 3
assert isinstance(trades, DataFrame)
assert "pair" in trades.columns
assert "open_time" in trades.columns
def test_extract_trades_of_period():
pair = "UNITTEST/BTC"
timerange = TimeRange(None, 'line', 0, -1000)
data = load_pair_history(pair=pair, ticker_interval='1m',
datadir=None, timerange=timerange)
# timerange = 2017-11-14 06:07 - 2017-11-14 22:58:00
trades = DataFrame(
{'pair': [pair, pair, pair, pair],
'profit_percent': [0.0, 0.1, -0.2, -0.5],
'profit_abs': [0.0, 1, -2, -5],
'open_time': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime,
Arrow(2017, 11, 14, 9, 41, 0).datetime,
Arrow(2017, 11, 14, 14, 20, 0).datetime,
Arrow(2017, 11, 15, 3, 40, 0).datetime,
], utc=True
),
'close_time': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime,
Arrow(2017, 11, 14, 10, 41, 0).datetime,
Arrow(2017, 11, 14, 15, 25, 0).datetime,
Arrow(2017, 11, 15, 3, 55, 0).datetime,
], utc=True)
})
trades1 = extract_trades_of_period(data, trades)
# First and last trade are dropped as they are out of range
assert len(trades1) == 2
assert trades1.iloc[0].open_time == Arrow(2017, 11, 14, 9, 41, 0).datetime
assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime
assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime
assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime

View File

@ -187,6 +187,23 @@ def test_download_data_options() -> None:
assert args.exchange == 'binance' assert args.exchange == 'binance'
def test_plot_dataframe_options() -> None:
args = [
'--indicators1', 'sma10,sma100',
'--indicators2', 'macd,fastd,fastk',
'--plot-limit', '30',
'-p', 'UNITTEST/BTC',
]
arguments = Arguments(args, '')
arguments.common_scripts_options()
arguments.plot_dataframe_options()
pargs = arguments.parse_args(True)
assert pargs.indicators1 == "sma10,sma100"
assert pargs.indicators2 == "macd,fastd,fastk"
assert pargs.plot_limit == 30
assert pargs.pairs == "UNITTEST/BTC"
def test_check_int_positive() -> None: def test_check_int_positive() -> None:
assert Arguments.check_int_positive("3") == 3 assert Arguments.check_int_positive("3") == 3

View File

@ -11,9 +11,48 @@ from freqtrade.persistence import Trade, clean_dry_run_db, init
from freqtrade.tests.conftest import log_has from freqtrade.tests.conftest import log_has
@pytest.fixture(scope='function') def create_mock_trades(fee):
def init_persistence(default_conf): """
init(default_conf['db_url'], default_conf['dry_run']) Create some fake trades ...
"""
# Simulate dry_run entries
trade = Trade(
pair='ETH/BTC',
stake_amount=0.001,
amount=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
open_order_id='dry_run_buy_12345'
)
Trade.session.add(trade)
trade = Trade(
pair='ETC/BTC',
stake_amount=0.001,
amount=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
is_open=False,
open_order_id='dry_run_sell_12345'
)
Trade.session.add(trade)
# Simulate prod entry
trade = Trade(
pair='ETC/BTC',
stake_amount=0.001,
amount=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
open_order_id='prod_buy_12345'
)
Trade.session.add(trade)
def test_init_create_session(default_conf): def test_init_create_session(default_conf):
@ -671,45 +710,7 @@ def test_adjust_min_max_rates(fee):
@pytest.mark.usefixtures("init_persistence") @pytest.mark.usefixtures("init_persistence")
def test_get_open(default_conf, fee): def test_get_open(default_conf, fee):
# Simulate dry_run entries create_mock_trades(fee)
trade = Trade(
pair='ETH/BTC',
stake_amount=0.001,
amount=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
open_order_id='dry_run_buy_12345'
)
Trade.session.add(trade)
trade = Trade(
pair='ETC/BTC',
stake_amount=0.001,
amount=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
is_open=False,
open_order_id='dry_run_sell_12345'
)
Trade.session.add(trade)
# Simulate prod entry
trade = Trade(
pair='ETC/BTC',
stake_amount=0.001,
amount=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
open_order_id='prod_buy_12345'
)
Trade.session.add(trade)
assert len(Trade.get_open_trades()) == 2 assert len(Trade.get_open_trades()) == 2

View File

@ -0,0 +1,188 @@
from unittest.mock import MagicMock
from plotly import tools
import plotly.graph_objs as go
from copy import deepcopy
from freqtrade.arguments import TimeRange
from freqtrade.data import history
from freqtrade.data.btanalysis import load_backtest_data
from freqtrade.plot.plotting import (generate_graph, generate_plot_file,
generate_row, plot_trades)
from freqtrade.strategy.default_strategy import DefaultStrategy
from freqtrade.tests.conftest import log_has, log_has_re
def fig_generating_mock(fig, *args, **kwargs):
""" Return Fig - used to mock generate_row and plot_trades"""
return fig
def find_trace_in_fig_data(data, search_string: str):
matches = filter(lambda x: x.name == search_string, data)
return next(matches)
def generage_empty_figure():
return tools.make_subplots(
rows=3,
cols=1,
shared_xaxes=True,
row_width=[1, 1, 4],
vertical_spacing=0.0001,
)
def test_generate_row(default_conf, caplog):
pair = "UNITTEST/BTC"
timerange = TimeRange(None, 'line', 0, -1000)
data = history.load_pair_history(pair=pair, ticker_interval='1m',
datadir=None, timerange=timerange)
indicators1 = ["ema10"]
indicators2 = ["macd"]
# Generate buy/sell signals and indicators
strat = DefaultStrategy(default_conf)
data = strat.analyze_ticker(data, {'pair': pair})
fig = generage_empty_figure()
# Row 1
fig1 = generate_row(fig=deepcopy(fig), row=1, indicators=indicators1, data=data)
figure = fig1.layout.figure
ema10 = find_trace_in_fig_data(figure.data, "ema10")
assert isinstance(ema10, go.Scatter)
assert ema10.yaxis == "y"
fig2 = generate_row(fig=deepcopy(fig), row=3, indicators=indicators2, data=data)
figure = fig2.layout.figure
macd = find_trace_in_fig_data(figure.data, "macd")
assert isinstance(macd, go.Scatter)
assert macd.yaxis == "y3"
# No indicator found
fig3 = generate_row(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data)
assert fig == fig3
assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog.record_tuples)
def test_plot_trades():
fig1 = generage_empty_figure()
# nothing happens when no trades are available
fig = plot_trades(fig1, None)
assert fig == fig1
pair = "ADA/BTC"
filename = history.make_testdata_path(None) / "backtest-result_test.json"
trades = load_backtest_data(filename)
trades = trades.loc[trades['pair'] == pair]
fig = plot_trades(fig, trades)
figure = fig1.layout.figure
# Check buys - color, should be in first graph, ...
trade_buy = find_trace_in_fig_data(figure.data, "trade_buy")
assert isinstance(trade_buy, go.Scatter)
assert trade_buy.yaxis == 'y'
assert len(trades) == len(trade_buy.x)
assert trade_buy.marker.color == 'green'
trade_sell = find_trace_in_fig_data(figure.data, "trade_sell")
assert isinstance(trade_sell, go.Scatter)
assert trade_sell.yaxis == 'y'
assert len(trades) == len(trade_sell.x)
assert trade_sell.marker.color == 'red'
def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog):
row_mock = mocker.patch('freqtrade.plot.plotting.generate_row',
MagicMock(side_effect=fig_generating_mock))
trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
MagicMock(side_effect=fig_generating_mock))
pair = "UNITTEST/BTC"
timerange = TimeRange(None, 'line', 0, -1000)
data = history.load_pair_history(pair=pair, ticker_interval='1m',
datadir=None, timerange=timerange)
data['buy'] = 0
data['sell'] = 0
indicators1 = []
indicators2 = []
fig = generate_graph(pair=pair, data=data, trades=None,
indicators1=indicators1, indicators2=indicators2)
assert isinstance(fig, go.Figure)
assert fig.layout.title.text == pair
figure = fig.layout.figure
assert len(figure.data) == 2
# Candlesticks are plotted first
candles = find_trace_in_fig_data(figure.data, "Price")
assert isinstance(candles, go.Candlestick)
volume = find_trace_in_fig_data(figure.data, "Volume")
assert isinstance(volume, go.Bar)
assert row_mock.call_count == 2
assert trades_mock.call_count == 1
assert log_has("No buy-signals found.", caplog.record_tuples)
assert log_has("No sell-signals found.", caplog.record_tuples)
def test_generate_graph_no_trades(default_conf, mocker):
row_mock = mocker.patch('freqtrade.plot.plotting.generate_row',
MagicMock(side_effect=fig_generating_mock))
trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
MagicMock(side_effect=fig_generating_mock))
pair = 'UNITTEST/BTC'
timerange = TimeRange(None, 'line', 0, -1000)
data = history.load_pair_history(pair=pair, ticker_interval='1m',
datadir=None, timerange=timerange)
# Generate buy/sell signals and indicators
strat = DefaultStrategy(default_conf)
data = strat.analyze_ticker(data, {'pair': pair})
indicators1 = []
indicators2 = []
fig = generate_graph(pair=pair, data=data, trades=None,
indicators1=indicators1, indicators2=indicators2)
assert isinstance(fig, go.Figure)
assert fig.layout.title.text == pair
figure = fig.layout.figure
assert len(figure.data) == 6
# Candlesticks are plotted first
candles = find_trace_in_fig_data(figure.data, "Price")
assert isinstance(candles, go.Candlestick)
volume = find_trace_in_fig_data(figure.data, "Volume")
assert isinstance(volume, go.Bar)
buy = find_trace_in_fig_data(figure.data, "buy")
assert isinstance(buy, go.Scatter)
# All buy-signals should be plotted
assert int(data.buy.sum()) == len(buy.x)
sell = find_trace_in_fig_data(figure.data, "sell")
assert isinstance(sell, go.Scatter)
# All buy-signals should be plotted
assert int(data.sell.sum()) == len(sell.x)
assert find_trace_in_fig_data(figure.data, "BB lower")
assert find_trace_in_fig_data(figure.data, "BB upper")
assert row_mock.call_count == 2
assert trades_mock.call_count == 1
def test_generate_plot_file(mocker, caplog):
fig = generage_empty_figure()
plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock())
generate_plot_file(fig, "UNITTEST/BTC", "5m")
assert plot_mock.call_count == 1
assert plot_mock.call_args[0][0] == fig
assert (plot_mock.call_args_list[0][1]['filename']
== "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html")

View File

@ -1,5 +1,6 @@
# Include all requirements to run the bot. # Include all requirements to run the bot.
-r requirements.txt -r requirements.txt
-r requirements-plot.txt
flake8==3.7.7 flake8==3.7.7
flake8-type-annotations==0.1.0 flake8-type-annotations==0.1.0

View File

@ -26,128 +26,40 @@ Example of usage:
""" """
import logging import logging
import sys import sys
from argparse import Namespace
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List from typing import Any, Dict, List
import pandas as pd import pandas as pd
import plotly.graph_objs as go
import pytz
from plotly import tools
from plotly.offline import plot
from freqtrade import persistence
from freqtrade.arguments import Arguments, TimeRange from freqtrade.arguments import Arguments, TimeRange
from freqtrade.data import history from freqtrade.data import history
from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data from freqtrade.data.btanalysis import load_trades, extract_trades_of_period
from freqtrade.exchange import Exchange
from freqtrade.optimize import setup_configuration from freqtrade.optimize import setup_configuration
from freqtrade.persistence import Trade from freqtrade.plot.plotting import (generate_graph,
from freqtrade.resolvers import StrategyResolver generate_plot_file)
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.state import RunMode from freqtrade.state import RunMode
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_CONF: Dict[str, Any] = {}
timeZone = pytz.UTC
def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange,
trades: pd.DataFrame = pd.DataFrame() datadir: Path, refresh_pairs: bool, live: bool):
if args.db_url:
persistence.init(args.db_url, clean_open_orders=False)
columns = ["pair", "profit", "open_time", "close_time",
"open_rate", "close_rate", "duration"]
for x in Trade.query.all():
print("date: {}".format(x.open_date))
trades = pd.DataFrame([(t.pair, t.calc_profit(),
t.open_date.replace(tzinfo=timeZone),
t.close_date.replace(tzinfo=timeZone) if t.close_date else None,
t.open_rate, t.close_rate,
t.close_date.timestamp() - t.open_date.timestamp()
if t.close_date else None)
for t in Trade.query.filter(Trade.pair.is_(pair)).all()],
columns=columns)
elif args.exportfilename:
file = Path(args.exportfilename)
if file.exists():
trades = load_backtest_data(file)
else:
trades = pd.DataFrame([], columns=BT_DATA_COLUMNS)
return trades
def generate_plot_file(fig, pair, ticker_interval, is_last) -> None:
"""
Generate a plot html file from pre populated fig plotly object
:return: None
"""
logger.info('Generate plot file for %s', pair)
pair_name = pair.replace("/", "_")
file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html'
Path("user_data/plots").mkdir(parents=True, exist_ok=True)
plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), auto_open=False)
if is_last:
plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False)
def get_trading_env(args: Namespace):
"""
Initalize freqtrade Exchange and Strategy, split pairs recieved in parameter
:return: Strategy
"""
global _CONF
# Load the configuration
_CONF.update(setup_configuration(args, RunMode.BACKTEST))
print(_CONF)
pairs = args.pairs.split(',')
if pairs is None:
logger.critical('Parameter --pairs mandatory;. E.g --pairs ETH/BTC,XRP/BTC')
exit()
# Load the strategy
try:
strategy = StrategyResolver(_CONF).strategy
exchange = Exchange(_CONF)
except AttributeError:
logger.critical(
'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"',
args.strategy
)
exit()
return [strategy, exchange, pairs]
def get_tickers_data(strategy, exchange, pairs: List[str], args):
""" """
Get tickers data for each pairs on live or local, option defined in args Get tickers data for each pairs on live or local, option defined in args
:return: dictinnary of tickers. output format: {'pair': tickersdata} :return: dictionary of tickers. output format: {'pair': tickersdata}
""" """
ticker_interval = strategy.ticker_interval ticker_interval = strategy.ticker_interval
timerange = Arguments.parse_timerange(args.timerange)
tickers = history.load_data( tickers = history.load_data(
datadir=Path(str(_CONF.get("datadir"))), datadir=datadir,
pairs=pairs, pairs=pairs,
ticker_interval=ticker_interval, ticker_interval=ticker_interval,
refresh_pairs=_CONF.get('refresh_pairs', False), refresh_pairs=refresh_pairs,
timerange=timerange, timerange=timerange,
exchange=Exchange(_CONF), exchange=exchange,
live=args.live, live=live,
) )
# No ticker found, impossible to download, len mismatch # No ticker found, impossible to download, len mismatch
@ -158,7 +70,7 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args):
if data.empty: if data.empty:
del tickers[pair] del tickers[pair]
logger.info( logger.info(
'An issue occured while retreiving datas of %s pair, please retry ' 'An issue occured while retreiving data of %s pair, please retry '
'using -l option for live or --refresh-pairs-cached', pair) 'using -l option for live or --refresh-pairs-cached', pair)
return tickers return tickers
@ -177,172 +89,61 @@ def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame:
return dataframe return dataframe
def extract_trades_of_period(dataframe, trades) -> pd.DataFrame: def analyse_and_plot_pairs(config: Dict[str, Any]):
""" """
Compare trades and backtested pair DataFrames to get trades performed on backtested period From arguments provided in cli:
:return: the DataFrame of a trades of period -Initialise backtest env
""" -Get tickers data
trades = trades.loc[trades['open_time'] >= dataframe.iloc[0]['date']] -Generate Dafaframes populated with indicators and signals
return trades -Load trades excecuted on same periods
-Generate Plotly plot objects
-Generate plot files
def generate_graph(
pair: str,
trades: pd.DataFrame,
data: pd.DataFrame,
indicators1: str,
indicators2: str
) -> tools.make_subplots:
"""
Generate the graph from the data generated by Backtesting or from DB
:param pair: Pair to Display on the graph
:param trades: All trades created
:param data: Dataframe
:indicators1: String Main plot indicators
:indicators2: String Sub plot indicators
:return: None :return: None
""" """
exchange_name = config.get('exchange', {}).get('name').title()
exchange = ExchangeResolver(exchange_name, config).exchange
# Define the graph strategy = StrategyResolver(config).strategy
fig = tools.make_subplots( if "pairs" in config:
rows=3, pairs = config["pairs"].split(',')
cols=1,
shared_xaxes=True,
row_width=[1, 1, 4],
vertical_spacing=0.0001,
)
fig['layout'].update(title=pair)
fig['layout']['yaxis1'].update(title='Price')
fig['layout']['yaxis2'].update(title='Volume')
fig['layout']['yaxis3'].update(title='Other')
fig['layout']['xaxis']['rangeslider'].update(visible=False)
# Common information
candles = go.Candlestick(
x=data.date,
open=data.open,
high=data.high,
low=data.low,
close=data.close,
name='Price'
)
df_buy = data[data['buy'] == 1]
buys = go.Scattergl(
x=df_buy.date,
y=df_buy.close,
mode='markers',
name='buy',
marker=dict(
symbol='triangle-up-dot',
size=9,
line=dict(width=1),
color='green',
)
)
df_sell = data[data['sell'] == 1]
sells = go.Scattergl(
x=df_sell.date,
y=df_sell.close,
mode='markers',
name='sell',
marker=dict(
symbol='triangle-down-dot',
size=9,
line=dict(width=1),
color='red',
)
)
trade_buys = go.Scattergl(
x=trades["open_time"],
y=trades["open_rate"],
mode='markers',
name='trade_buy',
marker=dict(
symbol='square-open',
size=11,
line=dict(width=2),
color='green'
)
)
trade_sells = go.Scattergl(
x=trades["close_time"],
y=trades["close_rate"],
mode='markers',
name='trade_sell',
marker=dict(
symbol='square-open',
size=11,
line=dict(width=2),
color='red'
)
)
# Row 1
fig.append_trace(candles, 1, 1)
if 'bb_lowerband' in data and 'bb_upperband' in data:
bb_lower = go.Scatter(
x=data.date,
y=data.bb_lowerband,
name='BB lower',
line={'color': 'rgba(255,255,255,0)'},
)
bb_upper = go.Scatter(
x=data.date,
y=data.bb_upperband,
name='BB upper',
fill="tonexty",
fillcolor="rgba(0,176,246,0.2)",
line={'color': 'rgba(255,255,255,0)'},
)
fig.append_trace(bb_lower, 1, 1)
fig.append_trace(bb_upper, 1, 1)
fig = generate_row(fig=fig, row=1, raw_indicators=indicators1, data=data)
fig.append_trace(buys, 1, 1)
fig.append_trace(sells, 1, 1)
fig.append_trace(trade_buys, 1, 1)
fig.append_trace(trade_sells, 1, 1)
# Row 2
volume = go.Bar(
x=data['date'],
y=data['volume'],
name='Volume'
)
fig.append_trace(volume, 2, 1)
# Row 3
fig = generate_row(fig=fig, row=3, raw_indicators=indicators2, data=data)
return fig
def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots:
"""
Generator all the indicator selected by the user for a specific row
"""
for indicator in raw_indicators.split(','):
if indicator in data:
scattergl = go.Scattergl(
x=data['date'],
y=data[indicator],
name=indicator
)
fig.append_trace(scattergl, row, 1)
else: else:
logger.info( pairs = config["exchange"]["pair_whitelist"]
'Indicator "%s" ignored. Reason: This indicator is not found '
'in your strategy.', # Set timerange to use
indicator timerange = Arguments.parse_timerange(config["timerange"])
ticker_interval = strategy.ticker_interval
tickers = get_tickers_data(strategy, exchange, pairs, timerange,
datadir=Path(str(config.get("datadir"))),
refresh_pairs=config.get('refresh_pairs', False),
live=config.get("live", False))
pair_counter = 0
for pair, data in tickers.items():
pair_counter += 1
logger.info("analyse pair %s", pair)
tickers = {}
tickers[pair] = data
dataframe = generate_dataframe(strategy, tickers, pair)
trades = load_trades(db_url=config["db_url"],
exportfilename=config["exportfilename"])
trades = trades.loc[trades['pair'] == pair]
trades = extract_trades_of_period(dataframe, trades)
fig = generate_graph(
pair=pair,
data=dataframe,
trades=trades,
indicators1=config["indicators1"].split(","),
indicators2=config["indicators2"].split(",")
) )
return fig generate_plot_file(fig, pair, ticker_interval)
logger.info('End of ploting process %s plots generated', pair_counter)
def plot_parse_args(args: List[str]) -> Namespace: def plot_parse_args(args: List[str]) -> Dict[str, Any]:
""" """
Parse args passed to the script Parse args passed to the script
:param args: Cli arguments :param args: Cli arguments
@ -355,49 +156,11 @@ def plot_parse_args(args: List[str]) -> Namespace:
arguments.backtesting_options() arguments.backtesting_options()
arguments.common_scripts_options() arguments.common_scripts_options()
arguments.plot_dataframe_options() arguments.plot_dataframe_options()
return arguments.parse_args() parsed_args = arguments.parse_args()
# Load the configuration
def analyse_and_plot_pairs(args: Namespace): config = setup_configuration(parsed_args, RunMode.BACKTEST)
""" return config
From arguments provided in cli:
-Initialise backtest env
-Get tickers data
-Generate Dafaframes populated with indicators and signals
-Load trades excecuted on same periods
-Generate Plotly plot objects
-Generate plot files
:return: None
"""
strategy, exchange, pairs = get_trading_env(args)
# Set timerange to use
timerange = Arguments.parse_timerange(args.timerange)
ticker_interval = strategy.ticker_interval
tickers = get_tickers_data(strategy, exchange, pairs, args)
pair_counter = 0
for pair, data in tickers.items():
pair_counter += 1
logger.info("analyse pair %s", pair)
tickers = {}
tickers[pair] = data
dataframe = generate_dataframe(strategy, tickers, pair)
trades = load_trades(args, pair, timerange)
trades = extract_trades_of_period(dataframe, trades)
fig = generate_graph(
pair=pair,
trades=trades,
data=dataframe,
indicators1=args.indicators1,
indicators2=args.indicators2
)
is_last = (False, True)[pair_counter == len(tickers)]
generate_plot_file(fig, pair, ticker_interval, is_last)
logger.info('End of ploting process %s plots generated', pair_counter)
def main(sysargv: List[str]) -> None: def main(sysargv: List[str]) -> None: