Merge pull request #1937 from xmatthias/feat/plot_module

move parts of scripts/plot_dataframe.py to main bot code

freqtrade/arguments.py

@@ -56,7 +56,7 @@ class Arguments(object):
 
         # Workaround issue in argparse with action='append' and default value
         # (see https://bugs.python.org/issue16399)
-        if parsed_arg.config is None and not no_default_config:
+        if not no_default_config and parsed_arg.config is None:
             parsed_arg.config = [constants.DEFAULT_CONFIG]
 
         return parsed_arg

freqtrade/configuration.py

@@ -103,6 +103,9 @@ class Configuration(object):
         # Load Optimize configurations
         config = self._load_optimize_config(config)
 
+        # Add plotting options if available
+        config = self._load_plot_config(config)
+
         # Set runmode
         if not self.runmode:
             # Handle real mode, infer dry/live from config
@@ -338,6 +341,26 @@ class Configuration(object):
 
         return config
 
+    def _load_plot_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Extract information for sys.argv Plotting configuration
+        :return: configuration as dictionary
+        """
+
+        self._args_to_config(config, argname='pairs',
+                             logstring='Using pairs {}')
+
+        self._args_to_config(config, argname='indicators1',
+                             logstring='Using indicators1: {}')
+
+        self._args_to_config(config, argname='indicators2',
+                             logstring='Using indicators2: {}')
+
+        self._args_to_config(config, argname='plot_limit',
+                             logstring='Limiting plot to: {}')
+
+        return config
+
     def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]:
         """
         Validate the configuration follow the Config Schema

freqtrade/data/btanalysis.py

@@ -1,12 +1,18 @@
 """
 Helpers when analyzing backtest data
 """
+import logging
 from pathlib import Path
 
 import numpy as np
 import pandas as pd
+import pytz
 
+from freqtrade import persistence
 from freqtrade.misc import json_load
+from freqtrade.persistence import Trade
+
+logger = logging.getLogger(__name__)
 
 # must align with columns in backtest.py
 BT_DATA_COLUMNS = ["pair", "profitperc", "open_time", "close_time", "index", "duration",
@@ -65,3 +71,48 @@ def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int
     df2 = df2.set_index('date')
     df_final = df2.resample(freq)[['pair']].count()
     return df_final[df_final['pair'] > max_open_trades]
+
+
+def load_trades(db_url: str = None, exportfilename: str = None) -> pd.DataFrame:
+    """
+    Load trades, either from a DB (using dburl) or via a backtest export file.
+    :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite)
+    :param exportfilename: Path to a file exported from backtesting
+    :returns: Dataframe containing Trades
+    """
+    timeZone = pytz.UTC
+
+    trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS)
+
+    if db_url:
+        persistence.init(db_url, clean_open_orders=False)
+        columns = ["pair", "profit", "open_time", "close_time",
+                   "open_rate", "close_rate", "duration"]
+
+        for x in Trade.query.all():
+            logger.info("date: {}".format(x.open_date))
+
+        trades = pd.DataFrame([(t.pair, t.calc_profit(),
+                                t.open_date.replace(tzinfo=timeZone),
+                                t.close_date.replace(tzinfo=timeZone) if t.close_date else None,
+                                t.open_rate, t.close_rate,
+                                t.close_date.timestamp() - t.open_date.timestamp()
+                                if t.close_date else None)
+                               for t in Trade.query.all()],
+                              columns=columns)
+
+    elif exportfilename:
+
+        trades = load_backtest_data(Path(exportfilename))
+
+    return trades
+
+
+def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame:
+    """
+    Compare trades and backtested pair DataFrames to get trades performed on backtested period
+    :return: the DataFrame of a trades of period
+    """
+    trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) &
+                        (trades['close_time'] <= dataframe.iloc[-1]['date'])]
+    return trades

freqtrade/plot/plotting.py

@@ -0,0 +1,221 @@
+import logging
+from typing import List
+
+import pandas as pd
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+try:
+    from plotly import tools
+    from plotly.offline import plot
+    import plotly.graph_objs as go
+except ImportError:
+    logger.exception("Module plotly not found \n Please install using `pip install plotly`")
+    exit(1)
+
+
+def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots:
+    """
+    Generator all the indicator selected by the user for a specific row
+    :param fig: Plot figure to append to
+    :param row: row number for this plot
+    :param indicators: List of indicators present in the dataframe
+    :param data: candlestick DataFrame
+    """
+    for indicator in indicators:
+        if indicator in data:
+            # TODO: Figure out why scattergl causes problems
+            scattergl = go.Scatter(
+                x=data['date'],
+                y=data[indicator].values,
+                mode='lines',
+                name=indicator
+            )
+            fig.append_trace(scattergl, row, 1)
+        else:
+            logger.info(
+                'Indicator "%s" ignored. Reason: This indicator is not found '
+                'in your strategy.',
+                indicator
+            )
+
+    return fig
+
+
+def plot_trades(fig, trades: pd.DataFrame):
+    """
+    Plot trades to "fig"
+    """
+    # Trades can be empty
+    if trades is not None and len(trades) > 0:
+        trade_buys = go.Scatter(
+            x=trades["open_time"],
+            y=trades["open_rate"],
+            mode='markers',
+            name='trade_buy',
+            marker=dict(
+                symbol='square-open',
+                size=11,
+                line=dict(width=2),
+                color='green'
+            )
+        )
+        # Create description for sell summarizing the trade
+        desc = trades.apply(lambda row: f"{round(row['profitperc'], 3)}%, {row['sell_reason']}, "
+                                        f"{row['duration']}min",
+                            axis=1)
+        trade_sells = go.Scatter(
+            x=trades["close_time"],
+            y=trades["close_rate"],
+            text=desc,
+            mode='markers',
+            name='trade_sell',
+            marker=dict(
+                symbol='square-open',
+                size=11,
+                line=dict(width=2),
+                color='red'
+            )
+        )
+        fig.append_trace(trade_buys, 1, 1)
+        fig.append_trace(trade_sells, 1, 1)
+    return fig
+
+
+def generate_graph(
+    pair: str,
+    data: pd.DataFrame,
+    trades: pd.DataFrame = None,
+    indicators1: List[str] = [],
+    indicators2: List[str] = [],
+) -> go.Figure:
+    """
+    Generate the graph from the data generated by Backtesting or from DB
+    Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators
+    :param pair: Pair to Display on the graph
+    :param data: OHLCV DataFrame containing indicators and buy/sell signals
+    :param trades: All trades created
+    :param indicators1: List containing Main plot indicators
+    :param indicators2: List containing Sub plot indicators
+    :return: None
+    """
+
+    # Define the graph
+    fig = tools.make_subplots(
+        rows=3,
+        cols=1,
+        shared_xaxes=True,
+        row_width=[1, 1, 4],
+        vertical_spacing=0.0001,
+    )
+    fig['layout'].update(title=pair)
+    fig['layout']['yaxis1'].update(title='Price')
+    fig['layout']['yaxis2'].update(title='Volume')
+    fig['layout']['yaxis3'].update(title='Other')
+    fig['layout']['xaxis']['rangeslider'].update(visible=False)
+
+    # Common information
+    candles = go.Candlestick(
+        x=data.date,
+        open=data.open,
+        high=data.high,
+        low=data.low,
+        close=data.close,
+        name='Price'
+    )
+    fig.append_trace(candles, 1, 1)
+
+    if 'buy' in data.columns:
+        df_buy = data[data['buy'] == 1]
+        if len(df_buy) > 0:
+            buys = go.Scatter(
+                x=df_buy.date,
+                y=df_buy.close,
+                mode='markers',
+                name='buy',
+                marker=dict(
+                    symbol='triangle-up-dot',
+                    size=9,
+                    line=dict(width=1),
+                    color='green',
+                )
+            )
+            fig.append_trace(buys, 1, 1)
+        else:
+            logger.warning("No buy-signals found.")
+
+    if 'sell' in data.columns:
+        df_sell = data[data['sell'] == 1]
+        if len(df_sell) > 0:
+            sells = go.Scatter(
+                x=df_sell.date,
+                y=df_sell.close,
+                mode='markers',
+                name='sell',
+                marker=dict(
+                    symbol='triangle-down-dot',
+                    size=9,
+                    line=dict(width=1),
+                    color='red',
+                )
+            )
+            fig.append_trace(sells, 1, 1)
+        else:
+            logger.warning("No sell-signals found.")
+
+    if 'bb_lowerband' in data and 'bb_upperband' in data:
+        bb_lower = go.Scattergl(
+            x=data.date,
+            y=data.bb_lowerband,
+            name='BB lower',
+            line={'color': 'rgba(255,255,255,0)'},
+        )
+        bb_upper = go.Scattergl(
+            x=data.date,
+            y=data.bb_upperband,
+            name='BB upper',
+            fill="tonexty",
+            fillcolor="rgba(0,176,246,0.2)",
+            line={'color': 'rgba(255,255,255,0)'},
+        )
+        fig.append_trace(bb_lower, 1, 1)
+        fig.append_trace(bb_upper, 1, 1)
+
+    # Add indicators to main plot
+    fig = generate_row(fig=fig, row=1, indicators=indicators1, data=data)
+
+    fig = plot_trades(fig, trades)
+
+    # Volume goes to row 2
+    volume = go.Bar(
+        x=data['date'],
+        y=data['volume'],
+        name='Volume'
+    )
+    fig.append_trace(volume, 2, 1)
+
+    # Add indicators to seperate row
+    fig = generate_row(fig=fig, row=3, indicators=indicators2, data=data)
+
+    return fig
+
+
+def generate_plot_file(fig, pair, ticker_interval) -> None:
+    """
+    Generate a plot html file from pre populated fig plotly object
+    :param fig: Plotly Figure to plot
+    :param pair: Pair to plot (used as filename and Plot title)
+    :param ticker_interval: Used as part of the filename
+    :return: None
+    """
+    logger.info('Generate plot file for %s', pair)
+
+    pair_name = pair.replace("/", "_")
+    file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html'
+
+    Path("user_data/plots").mkdir(parents=True, exist_ok=True)
+
+    plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)),
+         auto_open=False)

freqtrade/tests/conftest.py

@@ -151,6 +151,11 @@ def patch_coinmarketcap(mocker) -> None:
     )
 
 
+@pytest.fixture(scope='function')
+def init_persistence(default_conf):
+    persistence.init(default_conf['db_url'], default_conf['dry_run'])
+
+
 @pytest.fixture(scope="function")
 def default_conf():
     """ Returns validated configuration suitable for most tests """

freqtrade/tests/data/test_btanalysis.py

@@ -1,8 +1,15 @@
-import pytest
+from unittest.mock import MagicMock
-from pandas import DataFrame
 
-from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data
+from arrow import Arrow
-from freqtrade.data.history import make_testdata_path
+import pytest
+from pandas import DataFrame, to_datetime
+
+from freqtrade.arguments import TimeRange
+from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
+                                       extract_trades_of_period,
+                                       load_backtest_data, load_trades)
+from freqtrade.data.history import load_pair_history, make_testdata_path
+from freqtrade.tests.test_persistence import create_mock_trades
 
 
 def test_load_backtest_data():
@@ -19,3 +26,59 @@ def test_load_backtest_data():
 
     with pytest.raises(ValueError, match=r"File .* does not exist\."):
         load_backtest_data(str("filename") + "nofile")
+
+
+def test_load_trades_file(default_conf, fee, mocker):
+    # Real testing of load_backtest_data is done in test_load_backtest_data
+    lbt = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock())
+    filename = make_testdata_path(None) / "backtest-result_test.json"
+    load_trades(db_url=None, exportfilename=filename)
+    assert lbt.call_count == 1
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_load_trades_db(default_conf, fee, mocker):
+
+    create_mock_trades(fee)
+    # remove init so it does not init again
+    init_mock = mocker.patch('freqtrade.persistence.init', MagicMock())
+
+    trades = load_trades(db_url=default_conf['db_url'], exportfilename=None)
+    assert init_mock.call_count == 1
+    assert len(trades) == 3
+    assert isinstance(trades, DataFrame)
+    assert "pair" in trades.columns
+    assert "open_time" in trades.columns
+
+
+def test_extract_trades_of_period():
+    pair = "UNITTEST/BTC"
+    timerange = TimeRange(None, 'line', 0, -1000)
+
+    data = load_pair_history(pair=pair, ticker_interval='1m',
+                             datadir=None, timerange=timerange)
+
+    # timerange = 2017-11-14 06:07 - 2017-11-14 22:58:00
+    trades = DataFrame(
+        {'pair': [pair, pair, pair, pair],
+         'profit_percent': [0.0, 0.1, -0.2, -0.5],
+         'profit_abs': [0.0, 1, -2, -5],
+         'open_time': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime,
+                                   Arrow(2017, 11, 14, 9, 41, 0).datetime,
+                                   Arrow(2017, 11, 14, 14, 20, 0).datetime,
+                                   Arrow(2017, 11, 15, 3, 40, 0).datetime,
+                                   ], utc=True
+                                  ),
+         'close_time': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime,
+                                    Arrow(2017, 11, 14, 10, 41, 0).datetime,
+                                    Arrow(2017, 11, 14, 15, 25, 0).datetime,
+                                    Arrow(2017, 11, 15, 3, 55, 0).datetime,
+                                    ], utc=True)
+         })
+    trades1 = extract_trades_of_period(data, trades)
+    # First and last trade are dropped as they are out of range
+    assert len(trades1) == 2
+    assert trades1.iloc[0].open_time == Arrow(2017, 11, 14, 9, 41, 0).datetime
+    assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime
+    assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime
+    assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime

freqtrade/tests/test_arguments.py

@@ -187,6 +187,23 @@ def test_download_data_options() -> None:
     assert args.exchange == 'binance'
 
 
+def test_plot_dataframe_options() -> None:
+    args = [
+        '--indicators1', 'sma10,sma100',
+        '--indicators2', 'macd,fastd,fastk',
+        '--plot-limit', '30',
+        '-p', 'UNITTEST/BTC',
+    ]
+    arguments = Arguments(args, '')
+    arguments.common_scripts_options()
+    arguments.plot_dataframe_options()
+    pargs = arguments.parse_args(True)
+    assert pargs.indicators1 == "sma10,sma100"
+    assert pargs.indicators2 == "macd,fastd,fastk"
+    assert pargs.plot_limit == 30
+    assert pargs.pairs == "UNITTEST/BTC"
+
+
 def test_check_int_positive() -> None:
 
     assert Arguments.check_int_positive("3") == 3

freqtrade/tests/test_persistence.py

@@ -11,9 +11,48 @@ from freqtrade.persistence import Trade, clean_dry_run_db, init
 from freqtrade.tests.conftest import log_has
 
 
-@pytest.fixture(scope='function')
+def create_mock_trades(fee):
-def init_persistence(default_conf):
+    """
-    init(default_conf['db_url'], default_conf['dry_run'])
+    Create some fake trades ...
+    """
+    # Simulate dry_run entries
+    trade = Trade(
+        pair='ETH/BTC',
+        stake_amount=0.001,
+        amount=123.0,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.123,
+        exchange='bittrex',
+        open_order_id='dry_run_buy_12345'
+    )
+    Trade.session.add(trade)
+
+    trade = Trade(
+        pair='ETC/BTC',
+        stake_amount=0.001,
+        amount=123.0,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.123,
+        exchange='bittrex',
+        is_open=False,
+        open_order_id='dry_run_sell_12345'
+    )
+    Trade.session.add(trade)
+
+    # Simulate prod entry
+    trade = Trade(
+        pair='ETC/BTC',
+        stake_amount=0.001,
+        amount=123.0,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.123,
+        exchange='bittrex',
+        open_order_id='prod_buy_12345'
+    )
+    Trade.session.add(trade)
 
 
 def test_init_create_session(default_conf):
@@ -671,45 +710,7 @@ def test_adjust_min_max_rates(fee):
 @pytest.mark.usefixtures("init_persistence")
 def test_get_open(default_conf, fee):
 
-    # Simulate dry_run entries
+    create_mock_trades(fee)
-    trade = Trade(
-        pair='ETH/BTC',
-        stake_amount=0.001,
-        amount=123.0,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.123,
-        exchange='bittrex',
-        open_order_id='dry_run_buy_12345'
-    )
-    Trade.session.add(trade)
-
-    trade = Trade(
-        pair='ETC/BTC',
-        stake_amount=0.001,
-        amount=123.0,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.123,
-        exchange='bittrex',
-        is_open=False,
-        open_order_id='dry_run_sell_12345'
-    )
-    Trade.session.add(trade)
-
-    # Simulate prod entry
-    trade = Trade(
-        pair='ETC/BTC',
-        stake_amount=0.001,
-        amount=123.0,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.123,
-        exchange='bittrex',
-        open_order_id='prod_buy_12345'
-    )
-    Trade.session.add(trade)
-
     assert len(Trade.get_open_trades()) == 2
 
 

freqtrade/tests/test_plotting.py

@@ -0,0 +1,188 @@
+
+from unittest.mock import MagicMock
+
+from plotly import tools
+import plotly.graph_objs as go
+from copy import deepcopy
+
+from freqtrade.arguments import TimeRange
+from freqtrade.data import history
+from freqtrade.data.btanalysis import load_backtest_data
+from freqtrade.plot.plotting import (generate_graph, generate_plot_file,
+                                     generate_row, plot_trades)
+from freqtrade.strategy.default_strategy import DefaultStrategy
+from freqtrade.tests.conftest import log_has, log_has_re
+
+
+def fig_generating_mock(fig, *args, **kwargs):
+    """ Return Fig - used to mock generate_row and plot_trades"""
+    return fig
+
+
+def find_trace_in_fig_data(data, search_string: str):
+    matches = filter(lambda x: x.name == search_string, data)
+    return next(matches)
+
+
+def generage_empty_figure():
+    return tools.make_subplots(
+        rows=3,
+        cols=1,
+        shared_xaxes=True,
+        row_width=[1, 1, 4],
+        vertical_spacing=0.0001,
+    )
+
+
+def test_generate_row(default_conf, caplog):
+    pair = "UNITTEST/BTC"
+    timerange = TimeRange(None, 'line', 0, -1000)
+
+    data = history.load_pair_history(pair=pair, ticker_interval='1m',
+                                     datadir=None, timerange=timerange)
+    indicators1 = ["ema10"]
+    indicators2 = ["macd"]
+
+    # Generate buy/sell signals and indicators
+    strat = DefaultStrategy(default_conf)
+    data = strat.analyze_ticker(data, {'pair': pair})
+    fig = generage_empty_figure()
+
+    # Row 1
+    fig1 = generate_row(fig=deepcopy(fig), row=1, indicators=indicators1, data=data)
+    figure = fig1.layout.figure
+    ema10 = find_trace_in_fig_data(figure.data, "ema10")
+    assert isinstance(ema10, go.Scatter)
+    assert ema10.yaxis == "y"
+
+    fig2 = generate_row(fig=deepcopy(fig), row=3, indicators=indicators2, data=data)
+    figure = fig2.layout.figure
+    macd = find_trace_in_fig_data(figure.data, "macd")
+    assert isinstance(macd, go.Scatter)
+    assert macd.yaxis == "y3"
+
+    # No indicator found
+    fig3 = generate_row(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data)
+    assert fig == fig3
+    assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog.record_tuples)
+
+
+def test_plot_trades():
+    fig1 = generage_empty_figure()
+    # nothing happens when no trades are available
+    fig = plot_trades(fig1, None)
+    assert fig == fig1
+    pair = "ADA/BTC"
+    filename = history.make_testdata_path(None) / "backtest-result_test.json"
+    trades = load_backtest_data(filename)
+    trades = trades.loc[trades['pair'] == pair]
+
+    fig = plot_trades(fig, trades)
+    figure = fig1.layout.figure
+
+    # Check buys - color, should be in first graph, ...
+    trade_buy = find_trace_in_fig_data(figure.data, "trade_buy")
+    assert isinstance(trade_buy, go.Scatter)
+    assert trade_buy.yaxis == 'y'
+    assert len(trades) == len(trade_buy.x)
+    assert trade_buy.marker.color == 'green'
+
+    trade_sell = find_trace_in_fig_data(figure.data, "trade_sell")
+    assert isinstance(trade_sell, go.Scatter)
+    assert trade_sell.yaxis == 'y'
+    assert len(trades) == len(trade_sell.x)
+    assert trade_sell.marker.color == 'red'
+
+
+def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog):
+    row_mock = mocker.patch('freqtrade.plot.plotting.generate_row',
+                            MagicMock(side_effect=fig_generating_mock))
+    trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
+                               MagicMock(side_effect=fig_generating_mock))
+
+    pair = "UNITTEST/BTC"
+    timerange = TimeRange(None, 'line', 0, -1000)
+    data = history.load_pair_history(pair=pair, ticker_interval='1m',
+                                     datadir=None, timerange=timerange)
+    data['buy'] = 0
+    data['sell'] = 0
+
+    indicators1 = []
+    indicators2 = []
+    fig = generate_graph(pair=pair, data=data, trades=None,
+                         indicators1=indicators1, indicators2=indicators2)
+    assert isinstance(fig, go.Figure)
+    assert fig.layout.title.text == pair
+    figure = fig.layout.figure
+
+    assert len(figure.data) == 2
+    # Candlesticks are plotted first
+    candles = find_trace_in_fig_data(figure.data, "Price")
+    assert isinstance(candles, go.Candlestick)
+
+    volume = find_trace_in_fig_data(figure.data, "Volume")
+    assert isinstance(volume, go.Bar)
+
+    assert row_mock.call_count == 2
+    assert trades_mock.call_count == 1
+
+    assert log_has("No buy-signals found.", caplog.record_tuples)
+    assert log_has("No sell-signals found.", caplog.record_tuples)
+
+
+def test_generate_graph_no_trades(default_conf, mocker):
+    row_mock = mocker.patch('freqtrade.plot.plotting.generate_row',
+                            MagicMock(side_effect=fig_generating_mock))
+    trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades',
+                               MagicMock(side_effect=fig_generating_mock))
+    pair = 'UNITTEST/BTC'
+    timerange = TimeRange(None, 'line', 0, -1000)
+    data = history.load_pair_history(pair=pair, ticker_interval='1m',
+                                     datadir=None, timerange=timerange)
+
+    # Generate buy/sell signals and indicators
+    strat = DefaultStrategy(default_conf)
+    data = strat.analyze_ticker(data, {'pair': pair})
+
+    indicators1 = []
+    indicators2 = []
+    fig = generate_graph(pair=pair, data=data, trades=None,
+                         indicators1=indicators1, indicators2=indicators2)
+    assert isinstance(fig, go.Figure)
+    assert fig.layout.title.text == pair
+    figure = fig.layout.figure
+
+    assert len(figure.data) == 6
+    # Candlesticks are plotted first
+    candles = find_trace_in_fig_data(figure.data, "Price")
+    assert isinstance(candles, go.Candlestick)
+
+    volume = find_trace_in_fig_data(figure.data, "Volume")
+    assert isinstance(volume, go.Bar)
+
+    buy = find_trace_in_fig_data(figure.data, "buy")
+    assert isinstance(buy, go.Scatter)
+    # All buy-signals should be plotted
+    assert int(data.buy.sum()) == len(buy.x)
+
+    sell = find_trace_in_fig_data(figure.data, "sell")
+    assert isinstance(sell, go.Scatter)
+    # All buy-signals should be plotted
+    assert int(data.sell.sum()) == len(sell.x)
+
+    assert find_trace_in_fig_data(figure.data, "BB lower")
+    assert find_trace_in_fig_data(figure.data, "BB upper")
+
+    assert row_mock.call_count == 2
+    assert trades_mock.call_count == 1
+
+
+def test_generate_plot_file(mocker, caplog):
+    fig = generage_empty_figure()
+    plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock())
+    generate_plot_file(fig, "UNITTEST/BTC", "5m")
+
+    assert plot_mock.call_count == 1
+    assert plot_mock.call_args[0][0] == fig
+    assert (plot_mock.call_args_list[0][1]['filename']
+            == "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html")

requirements-dev.txt

@@ -1,5 +1,6 @@
 # Include all requirements to run the bot.
 -r requirements.txt
+-r requirements-plot.txt
 
 flake8==3.7.7
 flake8-type-annotations==0.1.0

scripts/plot_dataframe.py

@@ -26,128 +26,40 @@ Example of usage:
 """
 import logging
 import sys
-from argparse import Namespace
 from pathlib import Path
 from typing import Any, Dict, List
 
 import pandas as pd
-import plotly.graph_objs as go
-import pytz
-from plotly import tools
-from plotly.offline import plot
 
-from freqtrade import persistence
 from freqtrade.arguments import Arguments, TimeRange
 from freqtrade.data import history
-from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data
+from freqtrade.data.btanalysis import load_trades, extract_trades_of_period
-from freqtrade.exchange import Exchange
 from freqtrade.optimize import setup_configuration
-from freqtrade.persistence import Trade
+from freqtrade.plot.plotting import (generate_graph,
-from freqtrade.resolvers import StrategyResolver
+                                     generate_plot_file)
+from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.state import RunMode
 
 logger = logging.getLogger(__name__)
-_CONF: Dict[str, Any] = {}
-
-timeZone = pytz.UTC
 
 
-def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame:
+def get_tickers_data(strategy, exchange, pairs: List[str], timerange: TimeRange,
-    trades: pd.DataFrame = pd.DataFrame()
+                     datadir: Path, refresh_pairs: bool, live: bool):
-    if args.db_url:
-        persistence.init(args.db_url, clean_open_orders=False)
-
-        columns = ["pair", "profit", "open_time", "close_time",
-                   "open_rate", "close_rate", "duration"]
-
-        for x in Trade.query.all():
-            print("date: {}".format(x.open_date))
-
-        trades = pd.DataFrame([(t.pair, t.calc_profit(),
-                                t.open_date.replace(tzinfo=timeZone),
-                                t.close_date.replace(tzinfo=timeZone) if t.close_date else None,
-                                t.open_rate, t.close_rate,
-                                t.close_date.timestamp() - t.open_date.timestamp()
-                                if t.close_date else None)
-                               for t in Trade.query.filter(Trade.pair.is_(pair)).all()],
-                              columns=columns)
-
-    elif args.exportfilename:
-
-        file = Path(args.exportfilename)
-        if file.exists():
-            trades = load_backtest_data(file)
-
-        else:
-            trades = pd.DataFrame([], columns=BT_DATA_COLUMNS)
-
-    return trades
-
-
-def generate_plot_file(fig, pair, ticker_interval, is_last) -> None:
-    """
-    Generate a plot html file from pre populated fig plotly object
-    :return: None
-    """
-    logger.info('Generate plot file for %s', pair)
-
-    pair_name = pair.replace("/", "_")
-    file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html'
-
-    Path("user_data/plots").mkdir(parents=True, exist_ok=True)
-
-    plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), auto_open=False)
-    if is_last:
-        plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False)
-
-
-def get_trading_env(args: Namespace):
-    """
-    Initalize freqtrade Exchange and Strategy, split pairs recieved in parameter
-    :return: Strategy
-    """
-    global _CONF
-
-    # Load the configuration
-    _CONF.update(setup_configuration(args, RunMode.BACKTEST))
-    print(_CONF)
-
-    pairs = args.pairs.split(',')
-    if pairs is None:
-        logger.critical('Parameter --pairs mandatory;. E.g --pairs ETH/BTC,XRP/BTC')
-        exit()
-
-    # Load the strategy
-    try:
-        strategy = StrategyResolver(_CONF).strategy
-        exchange = Exchange(_CONF)
-    except AttributeError:
-        logger.critical(
-            'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"',
-            args.strategy
-        )
-        exit()
-
-    return [strategy, exchange, pairs]
-
-
-def get_tickers_data(strategy, exchange, pairs: List[str], args):
     """
     Get tickers data for each pairs on live or local, option defined in args
-    :return: dictinnary of tickers. output format: {'pair': tickersdata}
+    :return: dictionary of tickers. output format: {'pair': tickersdata}
     """
 
     ticker_interval = strategy.ticker_interval
-    timerange = Arguments.parse_timerange(args.timerange)
 
     tickers = history.load_data(
-        datadir=Path(str(_CONF.get("datadir"))),
+        datadir=datadir,
         pairs=pairs,
         ticker_interval=ticker_interval,
-        refresh_pairs=_CONF.get('refresh_pairs', False),
+        refresh_pairs=refresh_pairs,
         timerange=timerange,
-        exchange=Exchange(_CONF),
+        exchange=exchange,
-        live=args.live,
+        live=live,
     )
 
     # No ticker found, impossible to download, len mismatch
@@ -158,7 +70,7 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args):
         if data.empty:
             del tickers[pair]
             logger.info(
-                'An issue occured while retreiving datas of %s pair, please retry '
+                'An issue occured while retreiving data of %s pair, please retry '
                 'using -l option for live or --refresh-pairs-cached', pair)
     return tickers
 
@@ -177,172 +89,61 @@ def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame:
     return dataframe
 
 
-def extract_trades_of_period(dataframe, trades) -> pd.DataFrame:
+def analyse_and_plot_pairs(config: Dict[str, Any]):
     """
-    Compare trades and backtested pair DataFrames to get trades performed on backtested period
+    From arguments provided in cli:
-    :return: the DataFrame of a trades of period
+    -Initialise backtest env
-    """
+    -Get tickers data
-    trades = trades.loc[trades['open_time'] >= dataframe.iloc[0]['date']]
+    -Generate Dafaframes populated with indicators and signals
-    return trades
+    -Load trades excecuted on same periods
-
+    -Generate Plotly plot objects
-
+    -Generate plot files
-def generate_graph(
-                    pair: str,
-                    trades: pd.DataFrame,
-                    data: pd.DataFrame,
-                    indicators1: str,
-                    indicators2: str
-                ) -> tools.make_subplots:
-    """
-    Generate the graph from the data generated by Backtesting or from DB
-    :param pair: Pair to Display on the graph
-    :param trades: All trades created
-    :param data: Dataframe
-    :indicators1: String Main plot indicators
-    :indicators2: String Sub plot indicators
     :return: None
     """
+    exchange_name = config.get('exchange', {}).get('name').title()
+    exchange = ExchangeResolver(exchange_name, config).exchange
 
-    # Define the graph
+    strategy = StrategyResolver(config).strategy
-    fig = tools.make_subplots(
+    if "pairs" in config:
-        rows=3,
+        pairs = config["pairs"].split(',')
-        cols=1,
+    else:
-        shared_xaxes=True,
+        pairs = config["exchange"]["pair_whitelist"]
-        row_width=[1, 1, 4],
-        vertical_spacing=0.0001,
-    )
-    fig['layout'].update(title=pair)
-    fig['layout']['yaxis1'].update(title='Price')
-    fig['layout']['yaxis2'].update(title='Volume')
-    fig['layout']['yaxis3'].update(title='Other')
-    fig['layout']['xaxis']['rangeslider'].update(visible=False)
 
-    # Common information
+    # Set timerange to use
-    candles = go.Candlestick(
+    timerange = Arguments.parse_timerange(config["timerange"])
-        x=data.date,
+    ticker_interval = strategy.ticker_interval
-        open=data.open,
-        high=data.high,
-        low=data.low,
-        close=data.close,
-        name='Price'
-    )
 
-    df_buy = data[data['buy'] == 1]
+    tickers = get_tickers_data(strategy, exchange, pairs, timerange,
-    buys = go.Scattergl(
+                               datadir=Path(str(config.get("datadir"))),
-        x=df_buy.date,
+                               refresh_pairs=config.get('refresh_pairs', False),
-        y=df_buy.close,
+                               live=config.get("live", False))
-        mode='markers',
+    pair_counter = 0
-        name='buy',
+    for pair, data in tickers.items():
-        marker=dict(
+        pair_counter += 1
-            symbol='triangle-up-dot',
+        logger.info("analyse pair %s", pair)
-            size=9,
+        tickers = {}
-            line=dict(width=1),
+        tickers[pair] = data
-            color='green',
+        dataframe = generate_dataframe(strategy, tickers, pair)
+
+        trades = load_trades(db_url=config["db_url"],
+                             exportfilename=config["exportfilename"])
+        trades = trades.loc[trades['pair'] == pair]
+        trades = extract_trades_of_period(dataframe, trades)
+
+        fig = generate_graph(
+            pair=pair,
+            data=dataframe,
+            trades=trades,
+            indicators1=config["indicators1"].split(","),
+            indicators2=config["indicators2"].split(",")
         )
-    )
-    df_sell = data[data['sell'] == 1]
-    sells = go.Scattergl(
-        x=df_sell.date,
-        y=df_sell.close,
-        mode='markers',
-        name='sell',
-        marker=dict(
-            symbol='triangle-down-dot',
-            size=9,
-            line=dict(width=1),
-            color='red',
-        )
-    )
 
-    trade_buys = go.Scattergl(
+        generate_plot_file(fig, pair, ticker_interval)
-        x=trades["open_time"],
-        y=trades["open_rate"],
-        mode='markers',
-        name='trade_buy',
-        marker=dict(
-            symbol='square-open',
-            size=11,
-            line=dict(width=2),
-            color='green'
-        )
-    )
-    trade_sells = go.Scattergl(
-        x=trades["close_time"],
-        y=trades["close_rate"],
-        mode='markers',
-        name='trade_sell',
-        marker=dict(
-            symbol='square-open',
-            size=11,
-            line=dict(width=2),
-            color='red'
-        )
-    )
 
-    # Row 1
+    logger.info('End of ploting process %s plots generated', pair_counter)
-    fig.append_trace(candles, 1, 1)
-
-    if 'bb_lowerband' in data and 'bb_upperband' in data:
-        bb_lower = go.Scatter(
-            x=data.date,
-            y=data.bb_lowerband,
-            name='BB lower',
-            line={'color': 'rgba(255,255,255,0)'},
-        )
-        bb_upper = go.Scatter(
-            x=data.date,
-            y=data.bb_upperband,
-            name='BB upper',
-            fill="tonexty",
-            fillcolor="rgba(0,176,246,0.2)",
-            line={'color': 'rgba(255,255,255,0)'},
-        )
-        fig.append_trace(bb_lower, 1, 1)
-        fig.append_trace(bb_upper, 1, 1)
-
-    fig = generate_row(fig=fig, row=1, raw_indicators=indicators1, data=data)
-    fig.append_trace(buys, 1, 1)
-    fig.append_trace(sells, 1, 1)
-    fig.append_trace(trade_buys, 1, 1)
-    fig.append_trace(trade_sells, 1, 1)
-
-    # Row 2
-    volume = go.Bar(
-        x=data['date'],
-        y=data['volume'],
-        name='Volume'
-    )
-    fig.append_trace(volume, 2, 1)
-
-    # Row 3
-    fig = generate_row(fig=fig, row=3, raw_indicators=indicators2, data=data)
-
-    return fig
 
 
-def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots:
+def plot_parse_args(args: List[str]) -> Dict[str, Any]:
-    """
-    Generator all the indicator selected by the user for a specific row
-    """
-    for indicator in raw_indicators.split(','):
-        if indicator in data:
-            scattergl = go.Scattergl(
-                x=data['date'],
-                y=data[indicator],
-                name=indicator
-            )
-            fig.append_trace(scattergl, row, 1)
-        else:
-            logger.info(
-                'Indicator "%s" ignored. Reason: This indicator is not found '
-                'in your strategy.',
-                indicator
-            )
-
-    return fig
-
-
-def plot_parse_args(args: List[str]) -> Namespace:
     """
     Parse args passed to the script
     :param args: Cli arguments
@@ -355,49 +156,11 @@ def plot_parse_args(args: List[str]) -> Namespace:
     arguments.backtesting_options()
     arguments.common_scripts_options()
     arguments.plot_dataframe_options()
-    return arguments.parse_args()
+    parsed_args = arguments.parse_args()
 
-
+    # Load the configuration
-def analyse_and_plot_pairs(args: Namespace):
+    config = setup_configuration(parsed_args, RunMode.BACKTEST)
-    """
+    return config
-    From arguments provided in cli:
-    -Initialise backtest env
-    -Get tickers data
-    -Generate Dafaframes populated with indicators and signals
-    -Load trades excecuted on same periods
-    -Generate Plotly plot objects
-    -Generate plot files
-    :return: None
-    """
-    strategy, exchange, pairs = get_trading_env(args)
-    # Set timerange to use
-    timerange = Arguments.parse_timerange(args.timerange)
-    ticker_interval = strategy.ticker_interval
-
-    tickers = get_tickers_data(strategy, exchange, pairs, args)
-    pair_counter = 0
-    for pair, data in tickers.items():
-        pair_counter += 1
-        logger.info("analyse pair %s", pair)
-        tickers = {}
-        tickers[pair] = data
-        dataframe = generate_dataframe(strategy, tickers, pair)
-
-        trades = load_trades(args, pair, timerange)
-        trades = extract_trades_of_period(dataframe, trades)
-
-        fig = generate_graph(
-            pair=pair,
-            trades=trades,
-            data=dataframe,
-            indicators1=args.indicators1,
-            indicators2=args.indicators2
-        )
-
-        is_last = (False, True)[pair_counter == len(tickers)]
-        generate_plot_file(fig, pair, ticker_interval, is_last)
-
-    logger.info('End of ploting process %s plots generated', pair_counter)
 
 
 def main(sysargv: List[str]) -> None: