Merge pull request #1987 from freqtrade/plot_script_changes

Plot script changes

freqtrade/arguments.py

@@ -359,7 +359,7 @@ ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY +
                         "refresh_pairs", "live"])
 
 ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY +
-                    ["pairs", "timerange", "export", "exportfilename"])
+                    ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"])
 
 
 class TimeRange(NamedTuple):

freqtrade/data/btanalysis.py

@@ -3,6 +3,7 @@ Helpers when analyzing backtest data
 """
 import logging
 from pathlib import Path
+from typing import Dict
 
 import numpy as np
 import pandas as pd
@@ -101,6 +102,18 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
     return trades
 
 
+def load_trades(config) -> pd.DataFrame:
+    """
+    Based on configuration option "trade_source":
+    * loads data from DB (using `db_url`)
+    * loads data from backtestfile (using `exportfilename`)
+    """
+    if config["trade_source"] == "DB":
+        return load_trades_from_db(config["db_url"])
+    elif config["trade_source"] == "file":
+        return load_backtest_data(Path(config["exportfilename"]))
+
+
 def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame:
     """
     Compare trades and backtested pair DataFrames to get trades performed on backtested period
@@ -109,3 +122,34 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> p
     trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) &
                         (trades['close_time'] <= dataframe.iloc[-1]['date'])]
     return trades
+
+
+def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "close"):
+    """
+    Combine multiple dataframes "column"
+    :param tickers: Dict of Dataframes, dict key should be pair.
+    :param column: Column in the original dataframes to use
+    :return: DataFrame with the column renamed to the dict key, and a column
+        named mean, containing the mean of all pairs.
+    """
+    df_comb = pd.concat([tickers[pair].set_index('date').rename(
+        {column: pair}, axis=1)[pair] for pair in tickers], axis=1)
+
+    df_comb['mean'] = df_comb.mean(axis=1)
+
+    return df_comb
+
+
+def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str) -> pd.DataFrame:
+    """
+    Adds a column `col_name` with the cumulative profit for the given trades array.
+    :param df: DataFrame with date index
+    :param trades: DataFrame containing trades (requires columns close_time and profitperc)
+    :return: Returns df with one additional column, col_name, containing the cumulative profit.
+    """
+    df[col_name] = trades.set_index('close_time')['profitperc'].cumsum()
+    # Set first value to 0
+    df.loc[df.iloc[0].name, col_name] = 0
+    # FFill to get continuous
+    df[col_name] = df[col_name].ffill()
+    return df

freqtrade/misc.py

@@ -5,10 +5,8 @@ import gzip
 import logging
 import re
 from datetime import datetime
-from typing import Dict
 
 import numpy as np
-from pandas import DataFrame
 import rapidjson
 
 
@@ -41,24 +39,6 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray:
     return dates.dt.to_pydatetime()
 
 
-def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray:
-    """
-    Return dates from Dataframe
-    :param dfs: Dict with format pair: pair_data
-    :return: List of dates
-    """
-    alldates = {}
-    for pair, pair_data in dfs.items():
-        dates = datesarray_to_datetimearray(pair_data['date'])
-        for date in dates:
-            alldates[date] = 1
-    lst = []
-    for date, _ in alldates.items():
-        lst.append(date)
-    arr = np.array(lst)
-    return np.sort(arr, axis=0)
-
-
 def file_dump_json(filename, data, is_zip=False) -> None:
     """
     Dump JSON data into a file

freqtrade/plot/plotting.py

@@ -1,8 +1,15 @@
 import logging
-from typing import List
+from pathlib import Path
+from typing import Dict, List, Optional
 
 import pandas as pd
-from pathlib import Path
+
+from freqtrade.arguments import Arguments
+from freqtrade.data import history
+from freqtrade.data.btanalysis import (combine_tickers_with_mean,
+                                       create_cum_profit, load_trades)
+from freqtrade.exchange import Exchange
+from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +23,46 @@ except ImportError:
     exit(1)
 
 
-def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots:
+def init_plotscript(config):
+    """
+    Initialize objects needed for plotting
+    :return: Dict with tickers, trades, pairs and strategy
+    """
+    exchange: Optional[Exchange] = None
+
+    # Exchange is only needed when downloading data!
+    if config.get("live", False) or config.get("refresh_pairs", False):
+        exchange = ExchangeResolver(config.get('exchange', {}).get('name'),
+                                    config).exchange
+
+    strategy = StrategyResolver(config).strategy
+    if "pairs" in config:
+        pairs = config["pairs"].split(',')
+    else:
+        pairs = config["exchange"]["pair_whitelist"]
+
+    # Set timerange to use
+    timerange = Arguments.parse_timerange(config["timerange"])
+
+    tickers = history.load_data(
+        datadir=Path(str(config.get("datadir"))),
+        pairs=pairs,
+        ticker_interval=config['ticker_interval'],
+        refresh_pairs=config.get('refresh_pairs', False),
+        timerange=timerange,
+        exchange=exchange,
+        live=config.get("live", False),
+    )
+
+    trades = load_trades(config)
+    return {"tickers": tickers,
+            "trades": trades,
+            "pairs": pairs,
+            "strategy": strategy,
+            }
+
+
+def add_indicators(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots:
     """
     Generator all the indicator selected by the user for a specific row
     :param fig: Plot figure to append to
@@ -44,9 +90,29 @@ def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.m
     return fig
 
 
-def plot_trades(fig, trades: pd.DataFrame):
+def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> tools.make_subplots:
     """
-    Plot trades to "fig"
+    Add profit-plot
+    :param fig: Plot figure to append to
+    :param row: row number for this plot
+    :param data: candlestick DataFrame
+    :param column: Column to use for plot
+    :param name: Name to use
+    :return: fig with added profit plot
+    """
+    profit = go.Scattergl(
+        x=data.index,
+        y=data[column],
+        name=name,
+    )
+    fig.append_trace(profit, row, 1)
+
+    return fig
+
+
+def plot_trades(fig, trades: pd.DataFrame) -> tools.make_subplots:
+    """
+    Add trades to "fig"
     """
     # Trades can be empty
     if trades is not None and len(trades) > 0:
@@ -86,13 +152,9 @@ def plot_trades(fig, trades: pd.DataFrame):
     return fig
 
 
-def generate_graph(
+def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFrame = None,
-    pair: str,
+                               indicators1: List[str] = [],
-    data: pd.DataFrame,
+                               indicators2: List[str] = [],) -> go.Figure:
-    trades: pd.DataFrame = None,
-    indicators1: List[str] = [],
-    indicators2: List[str] = [],
-) -> go.Figure:
     """
     Generate the graph from the data generated by Backtesting or from DB
     Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators
@@ -186,7 +248,7 @@ def generate_graph(
         fig.append_trace(bb_upper, 1, 1)
 
     # Add indicators to main plot
-    fig = generate_row(fig=fig, row=1, indicators=indicators1, data=data)
+    fig = add_indicators(fig=fig, row=1, indicators=indicators1, data=data)
 
     fig = plot_trades(fig, trades)
 
@@ -199,12 +261,54 @@ def generate_graph(
     fig.append_trace(volume, 2, 1)
 
     # Add indicators to seperate row
-    fig = generate_row(fig=fig, row=3, indicators=indicators2, data=data)
+    fig = add_indicators(fig=fig, row=3, indicators=indicators2, data=data)
 
     return fig
 
 
-def generate_plot_file(fig, pair, ticker_interval) -> None:
+def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame],
+                          trades: pd.DataFrame) -> go.Figure:
+    # Combine close-values for all pairs, rename columns to "pair"
+    df_comb = combine_tickers_with_mean(tickers, "close")
+
+    # Add combined cumulative profit
+    df_comb = create_cum_profit(df_comb, trades, 'cum_profit')
+
+    # Plot the pairs average close prices, and total profit growth
+    avgclose = go.Scattergl(
+        x=df_comb.index,
+        y=df_comb['mean'],
+        name='Avg close price',
+    )
+
+    fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1])
+    fig['layout'].update(title="Profit plot")
+
+    fig.append_trace(avgclose, 1, 1)
+    fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
+
+    for pair in pairs:
+        profit_col = f'cum_profit_{pair}'
+        df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col)
+
+        fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}")
+
+    return fig
+
+
+def generate_plot_filename(pair, ticker_interval) -> str:
+    """
+    Generate filenames per pair/ticker_interval to be used for storing plots
+    """
+    pair_name = pair.replace("/", "_")
+    file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html'
+
+    logger.info('Generate plot file for %s', pair)
+
+    return file_name
+
+
+def store_plot_file(fig, filename: str, auto_open: bool = False) -> None:
     """
     Generate a plot html file from pre populated fig plotly object
     :param fig: Plotly Figure to plot
@@ -212,12 +316,8 @@ def generate_plot_file(fig, pair, ticker_interval) -> None:
     :param ticker_interval: Used as part of the filename
     :return: None
     """
-    logger.info('Generate plot file for %s', pair)
-
-    pair_name = pair.replace("/", "_")
-    file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html'
 
     Path("user_data/plots").mkdir(parents=True, exist_ok=True)
 
-    plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)),
+    plot(fig, filename=str(Path('user_data/plots').joinpath(filename)),
-         auto_open=False)
+         auto_open=auto_open)

freqtrade/tests/data/test_btanalysis.py

@@ -1,14 +1,18 @@
 from unittest.mock import MagicMock
 
-from arrow import Arrow
 import pytest
+from arrow import Arrow
 from pandas import DataFrame, to_datetime
 
-from freqtrade.arguments import TimeRange
+from freqtrade.arguments import Arguments, TimeRange
 from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
+                                       combine_tickers_with_mean,
+                                       create_cum_profit,
                                        extract_trades_of_period,
-                                       load_backtest_data, load_trades_from_db)
+                                       load_backtest_data, load_trades,
-from freqtrade.data.history import load_pair_history, make_testdata_path
+                                       load_trades_from_db)
+from freqtrade.data.history import (load_data, load_pair_history,
+                                    make_testdata_path)
 from freqtrade.tests.test_persistence import create_mock_trades
 
 
@@ -74,3 +78,52 @@ def test_extract_trades_of_period():
     assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime
     assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime
     assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime
+
+
+def test_load_trades(default_conf, mocker):
+    db_mock = mocker.patch("freqtrade.data.btanalysis.load_trades_from_db", MagicMock())
+    bt_mock = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock())
+
+    default_conf['trade_source'] = "DB"
+    load_trades(default_conf)
+
+    assert db_mock.call_count == 1
+    assert bt_mock.call_count == 0
+
+    db_mock.reset_mock()
+    bt_mock.reset_mock()
+    default_conf['trade_source'] = "file"
+    default_conf['exportfilename'] = "testfile.json"
+    load_trades(default_conf)
+
+    assert db_mock.call_count == 0
+    assert bt_mock.call_count == 1
+
+
+def test_combine_tickers_with_mean():
+    pairs = ["ETH/BTC", "XLM/BTC"]
+    tickers = load_data(datadir=None,
+                        pairs=pairs,
+                        ticker_interval='5m'
+                        )
+    df = combine_tickers_with_mean(tickers)
+    assert isinstance(df, DataFrame)
+    assert "ETH/BTC" in df.columns
+    assert "XLM/BTC" in df.columns
+    assert "mean" in df.columns
+
+
+def test_create_cum_profit():
+    filename = make_testdata_path(None) / "backtest-result_test.json"
+    bt_data = load_backtest_data(filename)
+    timerange = Arguments.parse_timerange("20180110-20180112")
+
+    df = load_pair_history(pair="POWR/BTC", ticker_interval='5m',
+                           datadir=None, timerange=timerange)
+
+    cum_profits = create_cum_profit(df.set_index('date'),
+                                    bt_data[bt_data["pair"] == 'POWR/BTC'],
+                                    "cum_profits")
+    assert "cum_profits" in cum_profits.columns
+    assert cum_profits.iloc[0]['cum_profits'] == 0
+    assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005

freqtrade/tests/test_misc.py

@@ -4,10 +4,9 @@ import datetime
 from unittest.mock import MagicMock
 
 from freqtrade.data.converter import parse_ticker_dataframe
-from freqtrade.misc import (common_datearray, datesarray_to_datetimearray,
+from freqtrade.data.history import pair_data_filename
-                            file_dump_json, file_load_json, format_ms_time, shorten_date)
+from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json,
-from freqtrade.data.history import load_tickerdata_file, pair_data_filename
+                            file_load_json, format_ms_time, shorten_date)
-from freqtrade.strategy.default_strategy import DefaultStrategy
 
 
 def test_shorten_date() -> None:
@@ -32,20 +31,6 @@ def test_datesarray_to_datetimearray(ticker_history_list):
     assert date_len == 2
 
 
-def test_common_datearray(default_conf) -> None:
-    strategy = DefaultStrategy(default_conf)
-    tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
-    tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, "1m", pair="UNITTEST/BTC",
-                                                         fill_missing=True)}
-    dataframes = strategy.tickerdata_to_dataframe(tickerlist)
-
-    dates = common_datearray(dataframes)
-
-    assert dates.size == dataframes['UNITTEST/BTC']['date'].size
-    assert dates[0] == dataframes['UNITTEST/BTC']['date'][0]
-    assert dates[-1] == dataframes['UNITTEST/BTC']['date'].iloc[-1]
-
-
 def test_file_dump_json(mocker) -> None:
     file_open = mocker.patch('freqtrade.misc.open', MagicMock())
     json_dump = mocker.patch('rapidjson.dump', MagicMock())

freqtrade/tests/test_plotting.py

@@ -1,21 +1,24 @@
 
+from copy import deepcopy
 from unittest.mock import MagicMock
 
-from plotly import tools
 import plotly.graph_objs as go
-from copy import deepcopy
+from plotly import tools
 
-from freqtrade.arguments import TimeRange
+from freqtrade.arguments import Arguments, TimeRange
 from freqtrade.data import history
-from freqtrade.data.btanalysis import load_backtest_data
+from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data
-from freqtrade.plot.plotting import (generate_graph, generate_plot_file,
+from freqtrade.plot.plotting import (add_indicators, add_profit,
-                                     generate_row, plot_trades)
+                                     generate_candlestick_graph,
+                                     generate_plot_filename,
+                                     generate_profit_graph, init_plotscript,
+                                     plot_trades, store_plot_file)
 from freqtrade.strategy.default_strategy import DefaultStrategy
 from freqtrade.tests.conftest import log_has, log_has_re
 
 
 def fig_generating_mock(fig, *args, **kwargs):
-    """ Return Fig - used to mock generate_row and plot_trades"""
+    """ Return Fig - used to mock add_indicators and plot_trades"""
     return fig
 
 
@@ -34,7 +37,27 @@ def generage_empty_figure():
     )
 
 
-def test_generate_row(default_conf, caplog):
+def test_init_plotscript(default_conf, mocker):
+    default_conf['timerange'] = "20180110-20180112"
+    default_conf['trade_source'] = "file"
+    default_conf['ticker_interval'] = "5m"
+    default_conf["datadir"] = history.make_testdata_path(None)
+    default_conf['exportfilename'] = str(
+        history.make_testdata_path(None) / "backtest-result_test.json")
+    ret = init_plotscript(default_conf)
+    assert "tickers" in ret
+    assert "trades" in ret
+    assert "pairs" in ret
+    assert "strategy" in ret
+
+    default_conf['pairs'] = "POWR/BTC,XLM/BTC"
+    ret = init_plotscript(default_conf)
+    assert "tickers" in ret
+    assert "POWR/BTC" in ret["tickers"]
+    assert "XLM/BTC" in ret["tickers"]
+
+
+def test_add_indicators(default_conf, caplog):
     pair = "UNITTEST/BTC"
     timerange = TimeRange(None, 'line', 0, -1000)
 
@@ -49,20 +72,20 @@ def test_generate_row(default_conf, caplog):
     fig = generage_empty_figure()
 
     # Row 1
-    fig1 = generate_row(fig=deepcopy(fig), row=1, indicators=indicators1, data=data)
+    fig1 = add_indicators(fig=deepcopy(fig), row=1, indicators=indicators1, data=data)
     figure = fig1.layout.figure
     ema10 = find_trace_in_fig_data(figure.data, "ema10")
     assert isinstance(ema10, go.Scatter)
     assert ema10.yaxis == "y"
 
-    fig2 = generate_row(fig=deepcopy(fig), row=3, indicators=indicators2, data=data)
+    fig2 = add_indicators(fig=deepcopy(fig), row=3, indicators=indicators2, data=data)
     figure = fig2.layout.figure
     macd = find_trace_in_fig_data(figure.data, "macd")
     assert isinstance(macd, go.Scatter)
     assert macd.yaxis == "y3"
 
     # No indicator found
-    fig3 = generate_row(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data)
+    fig3 = add_indicators(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data)
     assert fig == fig3
     assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog.record_tuples)
 
@@ -95,8 +118,8 @@ def test_plot_trades(caplog):
     assert trade_sell.marker.color == 'red'
 
 
-def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog):
+def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, caplog):
-    row_mock = mocker.patch('freqtrade.plot.plotting.generate_row',
+    row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators',
                             MagicMock(side_effect=fig_generating_mock))
     trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
                                MagicMock(side_effect=fig_generating_mock))
@@ -110,8 +133,8 @@ def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog):
 
     indicators1 = []
     indicators2 = []
-    fig = generate_graph(pair=pair, data=data, trades=None,
+    fig = generate_candlestick_graph(pair=pair, data=data, trades=None,
-                         indicators1=indicators1, indicators2=indicators2)
+                                     indicators1=indicators1, indicators2=indicators2)
     assert isinstance(fig, go.Figure)
     assert fig.layout.title.text == pair
     figure = fig.layout.figure
@@ -131,8 +154,8 @@ def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog):
     assert log_has("No sell-signals found.", caplog.record_tuples)
 
 
-def test_generate_graph_no_trades(default_conf, mocker):
+def test_generate_candlestick_graph_no_trades(default_conf, mocker):
-    row_mock = mocker.patch('freqtrade.plot.plotting.generate_row',
+    row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators',
                             MagicMock(side_effect=fig_generating_mock))
     trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
                                MagicMock(side_effect=fig_generating_mock))
@@ -147,8 +170,8 @@ def test_generate_graph_no_trades(default_conf, mocker):
 
     indicators1 = []
     indicators2 = []
-    fig = generate_graph(pair=pair, data=data, trades=None,
+    fig = generate_candlestick_graph(pair=pair, data=data, trades=None,
-                         indicators1=indicators1, indicators2=indicators2)
+                                     indicators1=indicators1, indicators2=indicators2)
     assert isinstance(fig, go.Figure)
     assert fig.layout.title.text == pair
     figure = fig.layout.figure
@@ -178,12 +201,68 @@ def test_generate_graph_no_trades(default_conf, mocker):
     assert trades_mock.call_count == 1
 
 
+def test_generate_Plot_filename():
+    fn = generate_plot_filename("UNITTEST/BTC", "5m")
+    assert fn == "freqtrade-plot-UNITTEST_BTC-5m.html"
+
+
 def test_generate_plot_file(mocker, caplog):
     fig = generage_empty_figure()
     plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock())
-    generate_plot_file(fig, "UNITTEST/BTC", "5m")
+    store_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html")
 
     assert plot_mock.call_count == 1
     assert plot_mock.call_args[0][0] == fig
     assert (plot_mock.call_args_list[0][1]['filename']
             == "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html")
+
+
+def test_add_profit():
+    filename = history.make_testdata_path(None) / "backtest-result_test.json"
+    bt_data = load_backtest_data(filename)
+    timerange = Arguments.parse_timerange("20180110-20180112")
+
+    df = history.load_pair_history(pair="POWR/BTC", ticker_interval='5m',
+                                   datadir=None, timerange=timerange)
+    fig = generage_empty_figure()
+
+    cum_profits = create_cum_profit(df.set_index('date'),
+                                    bt_data[bt_data["pair"] == 'POWR/BTC'],
+                                    "cum_profits")
+
+    fig1 = add_profit(fig, row=2, data=cum_profits, column='cum_profits', name='Profits')
+    figure = fig1.layout.figure
+    profits = find_trace_in_fig_data(figure.data, "Profits")
+    assert isinstance(profits, go.Scattergl)
+    assert profits.yaxis == "y2"
+
+
+def test_generate_profit_graph():
+    filename = history.make_testdata_path(None) / "backtest-result_test.json"
+    trades = load_backtest_data(filename)
+    timerange = Arguments.parse_timerange("20180110-20180112")
+    pairs = ["POWR/BTC", "XLM/BTC"]
+
+    tickers = history.load_data(datadir=None,
+                                pairs=pairs,
+                                ticker_interval='5m',
+                                timerange=timerange
+                                )
+    trades = trades[trades['pair'].isin(pairs)]
+
+    fig = generate_profit_graph(pairs, tickers, trades)
+    assert isinstance(fig, go.Figure)
+
+    assert fig.layout.title.text == "Profit plot"
+    figure = fig.layout.figure
+    assert len(figure.data) == 4
+
+    avgclose = find_trace_in_fig_data(figure.data, "Avg close price")
+    assert isinstance(avgclose, go.Scattergl)
+
+    profit = find_trace_in_fig_data(figure.data, "Profit")
+    assert isinstance(profit, go.Scattergl)
+
+    for pair in pairs:
+        profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}")
+        assert isinstance(profit_pair, go.Scattergl)

scripts/plot_dataframe.py

@@ -2,19 +2,7 @@
 """
 Script to display when the bot will buy on specific pair(s)
 
-Mandatory Cli parameters:
+Use `python plot_dataframe.py --help` to display the command line arguments
--p / --pairs: pair(s) to examine
-
-Option but recommended
--s / --strategy: strategy to use
-
-
-Optional Cli parameters
--d / --datadir: path to pair(s) backtest data
---timerange: specify what timerange of data to use.
--l / --live: Live, to download the latest ticker for the pair(s)
--db / --db-url: Show trades stored in database
-
 
 Indicators recommended
 Row 1: sma, ema3, ema5, ema10, ema50
@@ -26,18 +14,16 @@ Example of usage:
 """
 import logging
 import sys
-from pathlib import Path
 from typing import Any, Dict, List
 
 import pandas as pd
 
 from freqtrade.arguments import ARGS_PLOT_DATAFRAME, Arguments
-from freqtrade.data import history
+from freqtrade.data.btanalysis import extract_trades_of_period
-from freqtrade.data.btanalysis import (extract_trades_of_period,
-                                       load_backtest_data, load_trades_from_db)
 from freqtrade.optimize import setup_configuration
-from freqtrade.plot.plotting import generate_graph, generate_plot_file
+from freqtrade.plot.plotting import (init_plotscript, generate_candlestick_graph,
-from freqtrade.resolvers import ExchangeResolver, StrategyResolver
+                                     store_plot_file,
+                                     generate_plot_filename)
 from freqtrade.state import RunMode
 
 logger = logging.getLogger(__name__)
@@ -68,52 +54,29 @@ def analyse_and_plot_pairs(config: Dict[str, Any]):
     -Generate plot files
     :return: None
     """
-    exchange = ExchangeResolver(config.get('exchange', {}).get('name'), config).exchange
+    plot_elements = init_plotscript(config)
-
+    trades = plot_elements['trades']
-    strategy = StrategyResolver(config).strategy
-    if "pairs" in config:
-        pairs = config["pairs"].split(',')
-    else:
-        pairs = config["exchange"]["pair_whitelist"]
-
-    # Set timerange to use
-    timerange = Arguments.parse_timerange(config["timerange"])
-    ticker_interval = strategy.ticker_interval
-
-    tickers = history.load_data(
-        datadir=Path(str(config.get("datadir"))),
-        pairs=pairs,
-        ticker_interval=config['ticker_interval'],
-        refresh_pairs=config.get('refresh_pairs', False),
-        timerange=timerange,
-        exchange=exchange,
-        live=config.get("live", False),
-    )
 
     pair_counter = 0
-    for pair, data in tickers.items():
+    for pair, data in plot_elements["tickers"].items():
         pair_counter += 1
         logger.info("analyse pair %s", pair)
         tickers = {}
         tickers[pair] = data
-        dataframe = generate_dataframe(strategy, tickers, pair)
+        dataframe = generate_dataframe(plot_elements["strategy"], tickers, pair)
-        if config["trade_source"] == "DB":
-            trades = load_trades_from_db(config["db_url"])
-        elif config["trade_source"] == "file":
-            trades = load_backtest_data(Path(config["exportfilename"]))
 
-        trades = trades.loc[trades['pair'] == pair]
+        trades_pair = trades.loc[trades['pair'] == pair]
-        trades = extract_trades_of_period(dataframe, trades)
+        trades_pair = extract_trades_of_period(dataframe, trades_pair)
 
-        fig = generate_graph(
+        fig = generate_candlestick_graph(
             pair=pair,
             data=dataframe,
-            trades=trades,
+            trades=trades_pair,
             indicators1=config["indicators1"].split(","),
             indicators2=config["indicators2"].split(",")
         )
 
-        generate_plot_file(fig, pair, ticker_interval)
+        store_plot_file(fig, generate_plot_filename(pair, config['ticker_interval']))
 
     logger.info('End of ploting process %s plots generated', pair_counter)
 
@@ -130,7 +93,7 @@ def plot_parse_args(args: List[str]) -> Dict[str, Any]:
     parsed_args = arguments.parse_args()
 
     # Load the configuration
-    config = setup_configuration(parsed_args, RunMode.BACKTEST)
+    config = setup_configuration(parsed_args, RunMode.OTHER)
     return config
 
 

scripts/plot_profit.py

@@ -2,204 +2,39 @@
 """
 Script to display profits
 
-Mandatory Cli parameters:
+Use `python plot_profit.py --help` to display the command line arguments
--p / --pair: pair to examine
-
-Optional Cli parameters
--c / --config: specify configuration file
--s / --strategy: strategy to use
--d / --datadir: path to pair backtest data
---timerange: specify what timerange of data to use
---export-filename: Specify where the backtest export is located.
 """
-import json
 import logging
 import sys
-from argparse import Namespace
+from typing import Any, Dict, List
-from pathlib import Path
-from typing import List, Optional
 
-import numpy as np
+from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments
-import plotly.graph_objs as go
+from freqtrade.optimize import setup_configuration
-from plotly import tools
+from freqtrade.plot.plotting import init_plotscript, generate_profit_graph, store_plot_file
-from plotly.offline import plot
-
-from freqtrade.arguments import Arguments, ARGS_PLOT_PROFIT
-from freqtrade.configuration import Configuration
-from freqtrade.data import history
-from freqtrade.exchange import timeframe_to_seconds
-from freqtrade.misc import common_datearray
-from freqtrade.resolvers import StrategyResolver
 from freqtrade.state import RunMode
 
-
 logger = logging.getLogger(__name__)
 
 
-# data:: [ pair,      profit-%,  enter,         exit,        time, duration]
+def plot_profit(config: Dict[str, Any]) -> None:
-# data:: ["ETH/BTC", 0.0023975, "1515598200", "1515602100", "2018-01-10 07:30:00+00:00", 65]
-def make_profit_array(data: List, px: int, min_date: int,
-                      interval: str,
-                      filter_pairs: Optional[List] = None) -> np.ndarray:
-    pg = np.zeros(px)
-    filter_pairs = filter_pairs or []
-    # Go through the trades
-    # and make an total profit
-    # array
-    for trade in data:
-        pair = trade[0]
-        if filter_pairs and pair not in filter_pairs:
-            continue
-        profit = trade[1]
-        trade_sell_time = int(trade[3])
-
-        ix = define_index(min_date, trade_sell_time, interval)
-        if ix < px:
-            logger.debug('[%s]: Add profit %s on %s', pair, profit, trade[4])
-            pg[ix] += profit
-
-    # rewrite the pg array to go from
-    # total profits at each timeframe
-    # to accumulated profits
-    pa = 0
-    for x in range(0, len(pg)):
-        p = pg[x]  # Get current total percent
-        pa += p  # Add to the accumulated percent
-        pg[x] = pa  # write back to save memory
-
-    return pg
-
-
-def plot_profit(args: Namespace) -> None:
     """
     Plots the total profit for all pairs.
     Note, the profit calculation isn't realistic.
     But should be somewhat proportional, and therefor useful
     in helping out to find a good algorithm.
     """
+    plot_elements = init_plotscript(config)
+    trades = plot_elements['trades']
+    # Filter trades to relevant pairs
+    trades = trades[trades['pair'].isin(plot_elements["pairs"])]
 
-    # We need to use the same pairs, same ticker_interval
+    # Create an average close price of all the pairs that were involved.
-    # and same timeperiod as used in backtesting
-    # to match the tickerdata against the profits-results
-    timerange = Arguments.parse_timerange(args.timerange)
-
-    config = Configuration(args, RunMode.OTHER).get_config()
-
-    # Init strategy
-    try:
-        strategy = StrategyResolver({'strategy': config.get('strategy')}).strategy
-
-    except AttributeError:
-        logger.critical(
-            'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"',
-            config.get('strategy')
-        )
-        exit(1)
-
-    # Load the profits results
-    try:
-        filename = args.exportfilename
-        with open(filename) as file:
-            data = json.load(file)
-    except FileNotFoundError:
-        logger.critical(
-            'File "backtest-result.json" not found. This script require backtesting '
-            'results to run.\nPlease run a backtesting with the parameter --export.')
-        exit(1)
-
-    # Take pairs from the cli otherwise switch to the pair in the config file
-    if args.pairs:
-        filter_pairs = args.pairs
-        filter_pairs = filter_pairs.split(',')
-    else:
-        filter_pairs = config['exchange']['pair_whitelist']
-
-    ticker_interval = strategy.ticker_interval
-    pairs = config['exchange']['pair_whitelist']
-
-    if filter_pairs:
-        pairs = list(set(pairs) & set(filter_pairs))
-        logger.info('Filter, keep pairs %s' % pairs)
-
-    tickers = history.load_data(
-        datadir=Path(str(config.get('datadir'))),
-        pairs=pairs,
-        ticker_interval=ticker_interval,
-        refresh_pairs=False,
-        timerange=timerange
-    )
-    dataframes = strategy.tickerdata_to_dataframe(tickers)
-
-    # NOTE: the dataframes are of unequal length,
-    # 'dates' is an merged date array of them all.
-
-    dates = common_datearray(dataframes)
-    min_date = int(min(dates).timestamp())
-    max_date = int(max(dates).timestamp())
-    num_iterations = define_index(min_date, max_date, ticker_interval) + 1
-
-    # Make an average close price of all the pairs that was involved.
     # this could be useful to gauge the overall market trend
-    # We are essentially saying:
+    fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades)
-    #  array <- sum dataframes[*]['close'] / num_items dataframes
+    store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True)
-    #  FIX: there should be some onliner numpy/panda for this
-    avgclose = np.zeros(num_iterations)
-    num = 0
-    for pair, pair_data in dataframes.items():
-        close = pair_data['close']
-        maxprice = max(close)  # Normalize price to [0,1]
-        logger.info('Pair %s has length %s' % (pair, len(close)))
-        for x in range(0, len(close)):
-            avgclose[x] += close[x] / maxprice
-        # avgclose += close
-        num += 1
-    avgclose /= num
-
-    # make an profits-growth array
-    pg = make_profit_array(data, num_iterations, min_date, ticker_interval, filter_pairs)
-
-    #
-    # Plot the pairs average close prices, and total profit growth
-    #
-
-    avgclose = go.Scattergl(
-        x=dates,
-        y=avgclose,
-        name='Avg close price',
-    )
-
-    profit = go.Scattergl(
-        x=dates,
-        y=pg,
-        name='Profit',
-    )
-
-    fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1])
-
-    fig.append_trace(avgclose, 1, 1)
-    fig.append_trace(profit, 2, 1)
-
-    for pair in pairs:
-        pg = make_profit_array(data, num_iterations, min_date, ticker_interval, [pair])
-        pair_profit = go.Scattergl(
-            x=dates,
-            y=pg,
-            name=pair,
-        )
-        fig.append_trace(pair_profit, 3, 1)
-
-    plot(fig, filename=str(Path('user_data').joinpath('freqtrade-profit-plot.html')))
 
 
-def define_index(min_date: int, max_date: int, ticker_interval: str) -> int:
+def plot_parse_args(args: List[str]) -> Dict[str, Any]:
-    """
-    Return the index of a specific date
-    """
-    interval_seconds = timeframe_to_seconds(ticker_interval)
-    return int((max_date - min_date) / interval_seconds)
-
-
-def plot_parse_args(args: List[str]) -> Namespace:
     """
     Parse args passed to the script
     :param args: Cli arguments
@@ -208,7 +43,11 @@ def plot_parse_args(args: List[str]) -> Namespace:
     arguments = Arguments(args, 'Graph profits')
     arguments.build_args(optionlist=ARGS_PLOT_PROFIT)
 
-    return arguments.parse_args()
+    parsed_args = arguments.parse_args()
+
+    # Load the configuration
+    config = setup_configuration(parsed_args, RunMode.OTHER)
+    return config
 
 
 def main(sysargv: List[str]) -> None: