Merge pull request #838 from freqtrade/fix/plot-scripts

Fix/Improve plot scripts

.gitignore

@@ -8,6 +8,7 @@ logfile.txt
 hyperopt_trials.pickle
 user_data/
 freqtrade-plot.html
+freqtrade-profit-plot.html
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

freqtrade/arguments.py

@@ -2,6 +2,7 @@
 This module contains the argument manager class
 """
 
+import os
 import argparse
 import logging
 import re
@@ -123,8 +124,8 @@ class Arguments(object):
         )
         parser.add_argument(
             '-r', '--refresh-pairs-cached',
-            help='refresh the pairs files in tests/testdata with the latest data from the exchange. \
+            help='refresh the pairs files in tests/testdata with the latest data from the '
-                  Use it if you want to run your backtesting with up-to-date data.',
+                 'exchange. Use it if you want to run your backtesting with up-to-date data.',
             action='store_true',
             dest='refresh_pairs',
         )
@@ -140,11 +141,12 @@ class Arguments(object):
             '--export-filename',
             help='Save backtest results to this filename \
                   requires --export to be set as well\
-                  Example --export-filename=backtest_today.json\
+                  Example --export-filename=user_data/backtest_data/backtest_today.json\
-                  (default: %(default)s',
+                  (default: %(default)s)',
             type=str,
-            default='backtest-result.json',
+            default=os.path.join('user_data', 'backtest_data', 'backtest-result.json'),
             dest='exportfilename',
+            metavar='PATH',
         )
 
     @staticmethod
@@ -220,8 +222,8 @@ class Arguments(object):
         self.hyperopt_options(hyperopt_cmd)
 
     @staticmethod
-    def parse_timerange(text: Optional[str]) -> Optional[Tuple[Tuple,
+    def parse_timerange(text: Optional[str]) -> \
-                                                         Optional[int], Optional[int]]]:
+            Optional[Tuple[Tuple, Optional[int], Optional[int]]]:
         """
         Parse the value of the argument --timerange to determine what is the range desired
         :param text: value from --timerange

scripts/plot_dataframe.py

@@ -5,28 +5,37 @@ Script to display when the bot will buy a specific pair
 Mandatory Cli parameters:
 -p / --pair: pair to examine
 
-Optional Cli parameters
+Option but recommended
 -s / --strategy: strategy to use
+
+
+Optional Cli parameters
 -d / --datadir: path to pair backtest data
 --timerange: specify what timerange of data to use.
 -l / --live: Live, to download the latest ticker for the pair
 -db / --db-url: Show trades stored in database
+
+
+Indicators recommended
+Row 1: sma, ema3, ema5, ema10, ema50
+Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk
+
+Example of usage:
+> python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3
+--indicators2 fastk,fastd
 """
 import logging
+import os
 import sys
 from argparse import Namespace
-
+from typing import Dict, List, Any
-from typing import List
+from sqlalchemy import create_engine
-
 from plotly import tools
 from plotly.offline import plot
 import plotly.graph_objs as go
-
-from typing import Dict, List, Any
-from sqlalchemy import create_engine
-
 from freqtrade.arguments import Arguments
 from freqtrade.analyze import Analyze
+from freqtrade.optimize.backtesting import setup_configuration
 from freqtrade import exchange
 import freqtrade.optimize as optimize
 from freqtrade import persistence
@@ -35,17 +44,35 @@ from freqtrade.persistence import Trade
 logger = logging.getLogger(__name__)
 _CONF: Dict[str, Any] = {}
 
+
 def plot_analyzed_dataframe(args: Namespace) -> None:
     """
     Calls analyze() and plots the returned dataframe
     :return: None
     """
-    pair = args.pair.replace('-', '_')
+
+    # Load the configuration
+    config = setup_configuration(args)
+
+    # Set the pair to audit
+    pair = args.pair
+
+    if pair is None:
+        logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC')
+        exit()
+
+    if '/' not in pair:
+        logger.critical('--pair format must be XXX/YYY')
+        exit()
+
+
+    # Set timerange to use
     timerange = Arguments.parse_timerange(args.timerange)
 
-    # Init strategy
+    # Load the strategy
     try:
-        analyze = Analyze({'strategy': args.strategy})
+        analyze = Analyze(config)
+        exchange.init(config)
     except AttributeError:
         logger.critical(
             'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"',
@@ -53,37 +80,76 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
         )
         exit()
 
-    tick_interval = analyze.strategy.ticker_interval
+    # Set the ticker to use
+    tick_interval = analyze.get_ticker_interval()
 
+    # Load pair tickers
     tickers = {}
     if args.live:
         logger.info('Downloading pair.')
-        # Init Bittrex to use public API
-        exchange.init({'key': '', 'secret': ''})
         tickers[pair] = exchange.get_ticker_history(pair, tick_interval)
     else:
         tickers = optimize.load_data(
             datadir=args.datadir,
             pairs=[pair],
             ticker_interval=tick_interval,
-            refresh_pairs=False,
+            refresh_pairs=config.get('refresh_pairs', False),
             timerange=timerange
         )
-    dataframes = analyze.tickerdata_to_dataframe(tickers)
-    dataframe = dataframes[pair]
-    dataframe = analyze.populate_buy_trend(dataframe)
-    dataframe = analyze.populate_sell_trend(dataframe)
 
+        # No ticker found, or impossible to download
+        if tickers == {}:
+            exit()
+
+    # Get trades already made from the DB
     trades = []
     if args.db_url:
         engine = create_engine('sqlite:///' + args.db_url)
         persistence.init(_CONF, engine)
         trades = Trade.query.filter(Trade.pair.is_(pair)).all()
 
+    dataframes = analyze.tickerdata_to_dataframe(tickers)
+    dataframe = dataframes[pair]
+    dataframe = analyze.populate_buy_trend(dataframe)
+    dataframe = analyze.populate_sell_trend(dataframe)
+
     if len(dataframe.index) > 750:
         logger.warning('Ticker contained more than 750 candles, clipping.')
-    data = dataframe.tail(750)
 
+    fig = generate_graph(
+        pair=pair,
+        trades=trades,
+        data=dataframe.tail(750),
+        args=args
+    )
+
+    plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html'))
+
+
+def generate_graph(pair, trades, data, args) -> tools.make_subplots:
+    """
+    Generate the graph from the data generated by Backtesting or from DB
+    :param pair: Pair to Display on the graph
+    :param trades: All trades created
+    :param data: Dataframe
+    :param args: sys.argv that contrains the two params indicators1, and indicators2
+    :return: None
+    """
+
+    # Define the graph
+    fig = tools.make_subplots(
+        rows=3,
+        cols=1,
+        shared_xaxes=True,
+        row_width=[1, 1, 4],
+        vertical_spacing=0.0001,
+    )
+    fig['layout'].update(title=pair)
+    fig['layout']['yaxis1'].update(title='Price')
+    fig['layout']['yaxis2'].update(title='Volume')
+    fig['layout']['yaxis3'].update(title='Other')
+
+    # Common information
     candles = go.Candlestick(
         x=data.date,
         open=data.open,
@@ -145,49 +211,67 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
         )
     )
 
-    bb_lower = go.Scatter(
+    # Row 1
-        x=data.date,
-        y=data.bb_lowerband,
-        name='BB lower',
-        line={'color': "transparent"},
-    )
-    bb_upper = go.Scatter(
-        x=data.date,
-        y=data.bb_upperband,
-        name='BB upper',
-        fill="tonexty",
-        fillcolor="rgba(0,176,246,0.2)",
-        line={'color': "transparent"},
-    )
-    macd = go.Scattergl(x=data['date'], y=data['macd'], name='MACD')
-    macdsignal = go.Scattergl(x=data['date'], y=data['macdsignal'], name='MACD signal')
-    volume = go.Bar(x=data['date'], y=data['volume'], name='Volume')
-
-    fig = tools.make_subplots(
-        rows=3,
-        cols=1,
-        shared_xaxes=True,
-        row_width=[1, 1, 4],
-        vertical_spacing=0.0001,
-    )
-
     fig.append_trace(candles, 1, 1)
-    fig.append_trace(bb_lower, 1, 1)
+
-    fig.append_trace(bb_upper, 1, 1)
+    if 'bb_lowerband' in data and 'bb_upperband' in data:
+        bb_lower = go.Scatter(
+            x=data.date,
+            y=data.bb_lowerband,
+            name='BB lower',
+            line={'color': "transparent"},
+        )
+        bb_upper = go.Scatter(
+            x=data.date,
+            y=data.bb_upperband,
+            name='BB upper',
+            fill="tonexty",
+            fillcolor="rgba(0,176,246,0.2)",
+            line={'color': "transparent"},
+        )
+        fig.append_trace(bb_lower, 1, 1)
+        fig.append_trace(bb_upper, 1, 1)
+
+    fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data)
     fig.append_trace(buys, 1, 1)
     fig.append_trace(sells, 1, 1)
-    fig.append_trace(volume, 2, 1)
-    fig.append_trace(macd, 3, 1)
-    fig.append_trace(macdsignal, 3, 1)
     fig.append_trace(trade_buys, 1, 1)
     fig.append_trace(trade_sells, 1, 1)
 
-    fig['layout'].update(title=args.pair)
+    # Row 2
-    fig['layout']['yaxis1'].update(title='Price')
+    volume = go.Bar(
-    fig['layout']['yaxis2'].update(title='Volume')
+        x=data['date'],
-    fig['layout']['yaxis3'].update(title='MACD')
+        y=data['volume'],
+        name='Volume'
+    )
+    fig.append_trace(volume, 2, 1)
 
-    plot(fig, filename='freqtrade-plot.html')
+    # Row 3
+    fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data)
+
+    return fig
+
+
+def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots:
+    """
+    Generator all the indicator selected by the user for a specific row
+    """
+    for indicator in raw_indicators.split(','):
+        if indicator in data:
+            scattergl = go.Scattergl(
+                x=data['date'],
+                y=data[indicator],
+                name=indicator
+            )
+            fig.append_trace(scattergl, row, 1)
+        else:
+            logger.info(
+                'Indicator "%s" ignored. Reason: This indicator is not found '
+                'in your strategy.',
+                indicator
+            )
+
+    return fig
 
 
 def plot_parse_args(args: List[str]) -> Namespace:
@@ -198,6 +282,24 @@ def plot_parse_args(args: List[str]) -> Namespace:
     """
     arguments = Arguments(args, 'Graph dataframe')
     arguments.scripts_options()
+    arguments.parser.add_argument(
+        '--indicators1',
+        help='Set indicators from your strategy you want in the first row of the graph. Separate '
+             'them with a coma. E.g: ema3,ema5 (default: %(default)s)',
+        type=str,
+        default='sma,ema3,ema5',
+        dest='indicators1',
+    )
+
+    arguments.parser.add_argument(
+        '--indicators2',
+        help='Set indicators from your strategy you want in the third row of the graph. Separate '
+             'them with a coma. E.g: fastd,fastk (default: %(default)s)',
+        type=str,
+        default='macd',
+        dest='indicators2',
+    )
+
     arguments.common_args_parser()
     arguments.optimizer_shared_options(arguments.parser)
     arguments.backtesting_options(arguments.parser)

scripts/plot_profit.py

@@ -8,9 +8,12 @@ Mandatory Cli parameters:
 Optional Cli parameters
 -c / --config: specify configuration file
 -s / --strategy: strategy to use
---timerange: specify what timerange of data to use.
+-d / --datadir: path to pair backtest data
+--timerange: specify what timerange of data to use
+--export-filename: Specify where the backtest export is located.
 """
 import logging
+import os
 import sys
 import json
 from argparse import Namespace
@@ -90,7 +93,18 @@ def plot_profit(args: Namespace) -> None:
             'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"',
             config.get('strategy')
         )
-        exit()
+        exit(0)
+
+    # Load the profits results
+    try:
+        filename = args.exportfilename
+        with open(filename) as file:
+            data = json.load(file)
+    except FileNotFoundError:
+        logger.critical(
+            'File "backtest-result.json" not found. This script require backtesting '
+            'results to run.\nPlease run a backtesting with the parameter --export.')
+        exit(0)
 
     # Take pairs from the cli otherwise switch to the pair in the config file
     if args.pair:
@@ -140,18 +154,7 @@ def plot_profit(args: Namespace) -> None:
         num += 1
     avgclose /= num
 
-    # Load the profits results
+    # make an profits-growth array
-    # And make an profits-growth array
-
-    try:
-        filename = 'backtest-result.json'
-        with open(filename) as file:
-            data = json.load(file)
-    except FileNotFoundError:
-        logger.critical('File "backtest-result.json" not found. This script require backtesting '
-                        'results to run.\nPlease run a backtesting with the parameter --export.')
-        exit(0)
-
     pg = make_profit_array(data, num_iterations, min_date, tick_interval, filter_pairs)
 
     #
@@ -184,7 +187,7 @@ def plot_profit(args: Namespace) -> None:
         )
         fig.append_trace(pair_profit, 3, 1)
 
-    plot(fig, filename='freqtrade-profit-plot.html')
+    plot(fig, filename=os.path.join('user_data', 'freqtrade-profit-plot.html'))
 
 
 def define_index(min_date: int, max_date: int, interval: str) -> int: