adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1987 from freqtrade/plot_script_changes

Browse Source 
Plot script changes
This commit is contained in:
Matthias
2019-07-03 06:43:34 +02:00
committed by GitHub
parent 687381f42c b3644f7fa0
commit 0908863e07
9 changed files with 359 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/arguments.py
View File

@@ -359,7 +359,7 @@ ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY +
"refresh_pairs", "live"])
ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY +
["pairs", "timerange", "export", "exportfilename"])
["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"])
class TimeRange(NamedTuple):

44
freqtrade/data/btanalysis.py
View File

@@ -3,6 +3,7 @@ Helpers when analyzing backtest data
"""
import logging
from pathlib import Path
from typing import Dict
import numpy as np
import pandas as pd
@@ -101,6 +102,18 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
return trades
def load_trades(config) -> pd.DataFrame:
"""
Based on configuration option "trade_source":
* loads data from DB (using `db_url`)
* loads data from backtestfile (using `exportfilename`)
"""
if config["trade_source"] == "DB":
return load_trades_from_db(config["db_url"])
elif config["trade_source"] == "file":
return load_backtest_data(Path(config["exportfilename"]))
def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame:
"""
Compare trades and backtested pair DataFrames to get trades performed on backtested period
@@ -109,3 +122,34 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> p
trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) &
(trades['close_time'] <= dataframe.iloc[-1]['date'])]
return trades
def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "close"):
"""
Combine multiple dataframes "column"
:param tickers: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return: DataFrame with the column renamed to the dict key, and a column
named mean, containing the mean of all pairs.
"""
df_comb = pd.concat([tickers[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in tickers], axis=1)
df_comb['mean'] = df_comb.mean(axis=1)
return df_comb
def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str) -> pd.DataFrame:
"""
Adds a column `col_name` with the cumulative profit for the given trades array.
:param df: DataFrame with date index
:param trades: DataFrame containing trades (requires columns close_time and profitperc)
:return: Returns df with one additional column, col_name, containing the cumulative profit.
"""
df[col_name] = trades.set_index('close_time')['profitperc'].cumsum()
# Set first value to 0
df.loc[df.iloc[0].name, col_name] = 0
# FFill to get continuous
df[col_name] = df[col_name].ffill()
return df

20
freqtrade/misc.py
View File

@@ -5,10 +5,8 @@ import gzip
import logging
import re
from datetime import datetime
from typing import Dict
import numpy as np
from pandas import DataFrame
import rapidjson
@@ -41,24 +39,6 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray:
return dates.dt.to_pydatetime()
def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray:
"""
Return dates from Dataframe
:param dfs: Dict with format pair: pair_data
:return: List of dates
"""
alldates = {}
for pair, pair_data in dfs.items():
dates = datesarray_to_datetimearray(pair_data['date'])
for date in dates:
alldates[date] = 1
lst = []
for date, _ in alldates.items():
lst.append(date)
arr = np.array(lst)
return np.sort(arr, axis=0)
def file_dump_json(filename, data, is_zip=False) -> None:
"""
Dump JSON data into a file

142
freqtrade/plot/plotting.py
View File

@@ -1,8 +1,15 @@
import logging
from typing import List
from pathlib import Path
from typing import Dict, List, Optional
import pandas as pd
from pathlib import Path
from freqtrade.arguments import Arguments
from freqtrade.data import history
from freqtrade.data.btanalysis import (combine_tickers_with_mean,
create_cum_profit, load_trades)
from freqtrade.exchange import Exchange
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
logger = logging.getLogger(__name__)
@@ -16,7 +23,46 @@ except ImportError:
exit(1)
def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots:
def init_plotscript(config):
"""
Initialize objects needed for plotting
:return: Dict with tickers, trades, pairs and strategy
"""
exchange: Optional[Exchange] = None
# Exchange is only needed when downloading data!
if config.get("live", False) or config.get("refresh_pairs", False):
exchange = ExchangeResolver(config.get('exchange', {}).get('name'),
config).exchange
strategy = StrategyResolver(config).strategy
if "pairs" in config:
pairs = config["pairs"].split(',')
else:
pairs = config["exchange"]["pair_whitelist"]
# Set timerange to use
timerange = Arguments.parse_timerange(config["timerange"])
tickers = history.load_data(
datadir=Path(str(config.get("datadir"))),
pairs=pairs,
ticker_interval=config['ticker_interval'],
refresh_pairs=config.get('refresh_pairs', False),
timerange=timerange,
exchange=exchange,
live=config.get("live", False),
)
trades = load_trades(config)
return {"tickers": tickers,
"trades": trades,
"pairs": pairs,
"strategy": strategy,
}
def add_indicators(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots:
"""
Generator all the indicator selected by the user for a specific row
:param fig: Plot figure to append to
@@ -44,9 +90,29 @@ def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.m
return fig
def plot_trades(fig, trades: pd.DataFrame):
def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> tools.make_subplots:
"""
Plot trades to "fig"
Add profit-plot
:param fig: Plot figure to append to
:param row: row number for this plot
:param data: candlestick DataFrame
:param column: Column to use for plot
:param name: Name to use
:return: fig with added profit plot
"""
profit = go.Scattergl(
x=data.index,
y=data[column],
name=name,
)
fig.append_trace(profit, row, 1)
return fig
def plot_trades(fig, trades: pd.DataFrame) -> tools.make_subplots:
"""
Add trades to "fig"
"""
# Trades can be empty
if trades is not None and len(trades) > 0:
@@ -86,13 +152,9 @@ def plot_trades(fig, trades: pd.DataFrame):
return fig
def generate_graph(
pair: str,
data: pd.DataFrame,
trades: pd.DataFrame = None,
indicators1: List[str] = [],
indicators2: List[str] = [],
) -> go.Figure:
def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFrame = None,
indicators1: List[str] = [],
indicators2: List[str] = [],) -> go.Figure:
"""
Generate the graph from the data generated by Backtesting or from DB
Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators
@@ -186,7 +248,7 @@ def generate_graph(
fig.append_trace(bb_upper, 1, 1)
# Add indicators to main plot
fig = generate_row(fig=fig, row=1, indicators=indicators1, data=data)
fig = add_indicators(fig=fig, row=1, indicators=indicators1, data=data)
fig = plot_trades(fig, trades)
@@ -199,12 +261,54 @@ def generate_graph(
fig.append_trace(volume, 2, 1)
# Add indicators to seperate row
fig = generate_row(fig=fig, row=3, indicators=indicators2, data=data)
fig = add_indicators(fig=fig, row=3, indicators=indicators2, data=data)
return fig
def generate_plot_file(fig, pair, ticker_interval) -> None:
def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame],
trades: pd.DataFrame) -> go.Figure:
# Combine close-values for all pairs, rename columns to "pair"
df_comb = combine_tickers_with_mean(tickers, "close")
# Add combined cumulative profit
df_comb = create_cum_profit(df_comb, trades, 'cum_profit')
# Plot the pairs average close prices, and total profit growth
avgclose = go.Scattergl(
x=df_comb.index,
y=df_comb['mean'],
name='Avg close price',
)
fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1])
fig['layout'].update(title="Profit plot")
fig.append_trace(avgclose, 1, 1)
fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
for pair in pairs:
profit_col = f'cum_profit_{pair}'
df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col)
fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}")
return fig
def generate_plot_filename(pair, ticker_interval) -> str:
"""
Generate filenames per pair/ticker_interval to be used for storing plots
"""
pair_name = pair.replace("/", "_")
file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html'
logger.info('Generate plot file for %s', pair)
return file_name
def store_plot_file(fig, filename: str, auto_open: bool = False) -> None:
"""
Generate a plot html file from pre populated fig plotly object
:param fig: Plotly Figure to plot
@@ -212,12 +316,8 @@ def generate_plot_file(fig, pair, ticker_interval) -> None:
:param ticker_interval: Used as part of the filename
:return: None
"""
logger.info('Generate plot file for %s', pair)
pair_name = pair.replace("/", "_")
file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html'
Path("user_data/plots").mkdir(parents=True, exist_ok=True)
plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)),
auto_open=False)
plot(fig, filename=str(Path('user_data/plots').joinpath(filename)),
auto_open=auto_open)

61
freqtrade/tests/data/test_btanalysis.py
View File

@@ -1,14 +1,18 @@
from unittest.mock import MagicMock
from arrow import Arrow
import pytest
from arrow import Arrow
from pandas import DataFrame, to_datetime
from freqtrade.arguments import TimeRange
from freqtrade.arguments import Arguments, TimeRange
from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
combine_tickers_with_mean,
create_cum_profit,
extract_trades_of_period,
load_backtest_data, load_trades_from_db)
from freqtrade.data.history import load_pair_history, make_testdata_path
load_backtest_data, load_trades,
load_trades_from_db)
from freqtrade.data.history import (load_data, load_pair_history,
make_testdata_path)
from freqtrade.tests.test_persistence import create_mock_trades
@@ -74,3 +78,52 @@ def test_extract_trades_of_period():
assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime
assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime
assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime
def test_load_trades(default_conf, mocker):
db_mock = mocker.patch("freqtrade.data.btanalysis.load_trades_from_db", MagicMock())
bt_mock = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock())
default_conf['trade_source'] = "DB"
load_trades(default_conf)
assert db_mock.call_count == 1
assert bt_mock.call_count == 0
db_mock.reset_mock()
bt_mock.reset_mock()
default_conf['trade_source'] = "file"
default_conf['exportfilename'] = "testfile.json"
load_trades(default_conf)
assert db_mock.call_count == 0
assert bt_mock.call_count == 1
def test_combine_tickers_with_mean():
pairs = ["ETH/BTC", "XLM/BTC"]
tickers = load_data(datadir=None,
pairs=pairs,
ticker_interval='5m'
)
df = combine_tickers_with_mean(tickers)
assert isinstance(df, DataFrame)
assert "ETH/BTC" in df.columns
assert "XLM/BTC" in df.columns
assert "mean" in df.columns
def test_create_cum_profit():
filename = make_testdata_path(None) / "backtest-result_test.json"
bt_data = load_backtest_data(filename)
timerange = Arguments.parse_timerange("20180110-20180112")
df = load_pair_history(pair="POWR/BTC", ticker_interval='5m',
datadir=None, timerange=timerange)
cum_profits = create_cum_profit(df.set_index('date'),
bt_data[bt_data["pair"] == 'POWR/BTC'],
"cum_profits")
assert "cum_profits" in cum_profits.columns
assert cum_profits.iloc[0]['cum_profits'] == 0
assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005

21
freqtrade/tests/test_misc.py
View File

@@ -4,10 +4,9 @@ import datetime
from unittest.mock import MagicMock
from freqtrade.data.converter import parse_ticker_dataframe
from freqtrade.misc import (common_datearray, datesarray_to_datetimearray,
file_dump_json, file_load_json, format_ms_time, shorten_date)
from freqtrade.data.history import load_tickerdata_file, pair_data_filename
from freqtrade.strategy.default_strategy import DefaultStrategy
from freqtrade.data.history import pair_data_filename
from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json,
file_load_json, format_ms_time, shorten_date)
def test_shorten_date() -> None:
@@ -32,20 +31,6 @@ def test_datesarray_to_datetimearray(ticker_history_list):
assert date_len == 2
def test_common_datearray(default_conf) -> None:
strategy = DefaultStrategy(default_conf)
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, "1m", pair="UNITTEST/BTC",
fill_missing=True)}
dataframes = strategy.tickerdata_to_dataframe(tickerlist)
dates = common_datearray(dataframes)
assert dates.size == dataframes['UNITTEST/BTC']['date'].size
assert dates[0] == dataframes['UNITTEST/BTC']['date'][0]
assert dates[-1] == dataframes['UNITTEST/BTC']['date'].iloc[-1]
def test_file_dump_json(mocker) -> None:
file_open = mocker.patch('freqtrade.misc.open', MagicMock())
json_dump = mocker.patch('rapidjson.dump', MagicMock())

119
freqtrade/tests/test_plotting.py
View File

@@ -1,21 +1,24 @@
from copy import deepcopy
from unittest.mock import MagicMock
from plotly import tools
import plotly.graph_objs as go
from copy import deepcopy
from plotly import tools
from freqtrade.arguments import TimeRange
from freqtrade.arguments import Arguments, TimeRange
from freqtrade.data import history
from freqtrade.data.btanalysis import load_backtest_data
from freqtrade.plot.plotting import (generate_graph, generate_plot_file,
generate_row, plot_trades)
from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data
from freqtrade.plot.plotting import (add_indicators, add_profit,
generate_candlestick_graph,
generate_plot_filename,
generate_profit_graph, init_plotscript,
plot_trades, store_plot_file)
from freqtrade.strategy.default_strategy import DefaultStrategy
from freqtrade.tests.conftest import log_has, log_has_re
def fig_generating_mock(fig, *args, **kwargs):
""" Return Fig - used to mock generate_row and plot_trades"""
""" Return Fig - used to mock add_indicators and plot_trades"""
return fig
@@ -34,7 +37,27 @@ def generage_empty_figure():
)
def test_generate_row(default_conf, caplog):
def test_init_plotscript(default_conf, mocker):
default_conf['timerange'] = "20180110-20180112"
default_conf['trade_source'] = "file"
default_conf['ticker_interval'] = "5m"
default_conf["datadir"] = history.make_testdata_path(None)
default_conf['exportfilename'] = str(
history.make_testdata_path(None) / "backtest-result_test.json")
ret = init_plotscript(default_conf)
assert "tickers" in ret
assert "trades" in ret
assert "pairs" in ret
assert "strategy" in ret
default_conf['pairs'] = "POWR/BTC,XLM/BTC"
ret = init_plotscript(default_conf)
assert "tickers" in ret
assert "POWR/BTC" in ret["tickers"]
assert "XLM/BTC" in ret["tickers"]
def test_add_indicators(default_conf, caplog):
pair = "UNITTEST/BTC"
timerange = TimeRange(None, 'line', 0, -1000)
@@ -49,20 +72,20 @@ def test_generate_row(default_conf, caplog):
fig = generage_empty_figure()
# Row 1
fig1 = generate_row(fig=deepcopy(fig), row=1, indicators=indicators1, data=data)
fig1 = add_indicators(fig=deepcopy(fig), row=1, indicators=indicators1, data=data)
figure = fig1.layout.figure
ema10 = find_trace_in_fig_data(figure.data, "ema10")
assert isinstance(ema10, go.Scatter)
assert ema10.yaxis == "y"
fig2 = generate_row(fig=deepcopy(fig), row=3, indicators=indicators2, data=data)
fig2 = add_indicators(fig=deepcopy(fig), row=3, indicators=indicators2, data=data)
figure = fig2.layout.figure
macd = find_trace_in_fig_data(figure.data, "macd")
assert isinstance(macd, go.Scatter)
assert macd.yaxis == "y3"
# No indicator found
fig3 = generate_row(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data)
fig3 = add_indicators(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data)
assert fig == fig3
assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog.record_tuples)
@@ -95,8 +118,8 @@ def test_plot_trades(caplog):
assert trade_sell.marker.color == 'red'
def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog):
row_mock = mocker.patch('freqtrade.plot.plotting.generate_row',
def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, caplog):
row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators',
MagicMock(side_effect=fig_generating_mock))
trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
MagicMock(side_effect=fig_generating_mock))
@@ -110,8 +133,8 @@ def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog):
indicators1 = []
indicators2 = []
fig = generate_graph(pair=pair, data=data, trades=None,
indicators1=indicators1, indicators2=indicators2)
fig = generate_candlestick_graph(pair=pair, data=data, trades=None,
indicators1=indicators1, indicators2=indicators2)
assert isinstance(fig, go.Figure)
assert fig.layout.title.text == pair
figure = fig.layout.figure
@@ -131,8 +154,8 @@ def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog):
assert log_has("No sell-signals found.", caplog.record_tuples)
def test_generate_graph_no_trades(default_conf, mocker):
row_mock = mocker.patch('freqtrade.plot.plotting.generate_row',
def test_generate_candlestick_graph_no_trades(default_conf, mocker):
row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators',
MagicMock(side_effect=fig_generating_mock))
trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
MagicMock(side_effect=fig_generating_mock))
@@ -147,8 +170,8 @@ def test_generate_graph_no_trades(default_conf, mocker):
indicators1 = []
indicators2 = []
fig = generate_graph(pair=pair, data=data, trades=None,
indicators1=indicators1, indicators2=indicators2)
fig = generate_candlestick_graph(pair=pair, data=data, trades=None,
indicators1=indicators1, indicators2=indicators2)
assert isinstance(fig, go.Figure)
assert fig.layout.title.text == pair
figure = fig.layout.figure
@@ -178,12 +201,68 @@ def test_generate_graph_no_trades(default_conf, mocker):
assert trades_mock.call_count == 1
def test_generate_Plot_filename():
fn = generate_plot_filename("UNITTEST/BTC", "5m")
assert fn == "freqtrade-plot-UNITTEST_BTC-5m.html"
def test_generate_plot_file(mocker, caplog):
fig = generage_empty_figure()
plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock())
generate_plot_file(fig, "UNITTEST/BTC", "5m")
store_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html")
assert plot_mock.call_count == 1
assert plot_mock.call_args[0][0] == fig
assert (plot_mock.call_args_list[0][1]['filename']
== "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html")
def test_add_profit():
filename = history.make_testdata_path(None) / "backtest-result_test.json"
bt_data = load_backtest_data(filename)
timerange = Arguments.parse_timerange("20180110-20180112")
df = history.load_pair_history(pair="POWR/BTC", ticker_interval='5m',
datadir=None, timerange=timerange)
fig = generage_empty_figure()
cum_profits = create_cum_profit(df.set_index('date'),
bt_data[bt_data["pair"] == 'POWR/BTC'],
"cum_profits")
fig1 = add_profit(fig, row=2, data=cum_profits, column='cum_profits', name='Profits')
figure = fig1.layout.figure
profits = find_trace_in_fig_data(figure.data, "Profits")
assert isinstance(profits, go.Scattergl)
assert profits.yaxis == "y2"
def test_generate_profit_graph():
filename = history.make_testdata_path(None) / "backtest-result_test.json"
trades = load_backtest_data(filename)
timerange = Arguments.parse_timerange("20180110-20180112")
pairs = ["POWR/BTC", "XLM/BTC"]
tickers = history.load_data(datadir=None,
pairs=pairs,
ticker_interval='5m',
timerange=timerange
)
trades = trades[trades['pair'].isin(pairs)]
fig = generate_profit_graph(pairs, tickers, trades)
assert isinstance(fig, go.Figure)
assert fig.layout.title.text == "Profit plot"
figure = fig.layout.figure
assert len(figure.data) == 4
avgclose = find_trace_in_fig_data(figure.data, "Avg close price")
assert isinstance(avgclose, go.Scattergl)
profit = find_trace_in_fig_data(figure.data, "Profit")
assert isinstance(profit, go.Scattergl)
for pair in pairs:
profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}")
assert isinstance(profit_pair, go.Scattergl)