Use dates on plot profit/dataframe

* plot_dataframe also support --timerange
* Both default to tkinter as matplotlib plotting backend

docs/plotting.md

@@ -17,6 +17,19 @@ python script/plot_dataframe.py -p BTC_ETH,BTC_LTC
 The -p pair argument, can be used to specify what
 pair you would like to plot.
 
+**Advanced use**
+
+To plot the current live price use the --live flag:
+```
+python scripts/plot_dataframe.py -p BTC_ETH --live
+```
+
+To plot a timerange (to zoom in):
+```
+python scripts/plot_dataframe.py -p BTC_ETH --timerange=100-200
+```
+Timerange doesn't work with live data.
+
 
 ## Plot profit
 
@@ -46,3 +59,11 @@ Example
 ```
 python python scripts/plot_profit.py --datadir ../freqtrade/freqtrade/tests/testdata-20171221/ -p BTC_LTC
 ```
+
+**When it goes wrong**
+
+*** Linux: Can't display**
+
+If you are inside an python environment, you might want to set the
+DISPLAY variable as so:
+$ DISPLAY=:0 python scripts/plot_dataframe.py

freqtrade/misc.py

@@ -5,8 +5,10 @@ import logging
 import time
 import os
 import re
+from datetime import datetime
 from typing import Any, Callable, Dict, List
 
+import numpy as np
 from jsonschema import Draft4Validator, validate
 from jsonschema.exceptions import ValidationError, best_match
 from wrapt import synchronized
@@ -16,11 +18,6 @@ from freqtrade import __version__
 logger = logging.getLogger(__name__)
 
 
-def file_dump_json(filename, data):
-    with open(filename, 'w') as fp:
-        json.dump(data, fp)
-
-
 class State(enum.Enum):
     RUNNING = 0
     STOPPED = 1
@@ -30,6 +27,44 @@ class State(enum.Enum):
 _STATE = State.STOPPED
 
 
+############################################
+# Used by scripts                          #
+# Matplotlib doesn't support ::datetime64, #
+# so we need to convert it into ::datetime #
+############################################
+
+def datesarray_to_datetimearray(dates):
+    """
+    Convert an pandas-array of timestamps into
+    An numpy-array of datetimes
+    :return: numpy-array of datetime
+    """
+    times = []
+    dates = dates.astype(datetime)
+    for i in range(0, dates.size):
+        date = dates[i].to_pydatetime()
+        times.append(date)
+    return np.array(times)
+
+
+def common_datearray(dfs):
+    alldates = {}
+    for pair, pair_data in dfs.items():
+        dates = datesarray_to_datetimearray(pair_data['date'])
+        for date in dates:
+            alldates[date] = 1
+    lst = []
+    for date, _ in alldates.items():
+        lst.append(date)
+    arr = np.array(lst)
+    return np.sort(arr, axis=0)
+
+
+def file_dump_json(filename, data):
+    with open(filename, 'w') as fp:
+        json.dump(data, fp)
+
+
 @synchronized
 def update_state(state: State) -> None:
     """
@@ -155,6 +190,15 @@ def parse_args(args: List[str], description: str):
     return parser.parse_args(args)
 
 
+def scripts_options(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        '-p', '--pair',
+        help='Show profits for only this pairs. Pairs are comma-separated.',
+        dest='pair',
+        default=None
+    )
+
+
 def backtesting_options(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         '-l', '--live',

scripts/plot_dataframe.py

@@ -1,23 +1,23 @@
 #!/usr/bin/env python3
 
 import sys
+import logging
 import argparse
-import matplotlib  # Install PYQT5 manually if you want to test this helper function
+import matplotlib
-matplotlib.use("Qt5Agg")
+import matplotlib.dates as mdates
 import matplotlib.pyplot as plt
 from freqtrade import exchange, analyze
-from freqtrade.misc import common_args_parser
+import freqtrade.misc as misc
+import freqtrade.optimize as optimize
+import freqtrade.analyze as analyze
+
+logger = logging.getLogger(__name__)
 
 
-def plot_parse_args(args ):
+def plot_parse_args(args):
-    parser = common_args_parser(description='Graph utility')
+    parser = misc.common_args_parser('Graph dataframe')
-    parser.add_argument(
+    misc.backtesting_options(parser)
-        '-p', '--pair',
+    misc.scripts_options(parser)
-        help = 'What currency pair',
-        dest = 'pair',
-        default = 'BTC_ETH',
-        type = str,
-    )
     return parser.parse_args(args)
 
 
@@ -28,11 +28,25 @@ def plot_analyzed_dataframe(args):
     :return: None
     """
     pair = args.pair
+    pairs = [pair]
+    timerange = misc.parse_timerange(args.timerange)
 
-    # Init Bittrex to use public API
+    tickers = {}
+    if args.live:
+        logger.info('Downloading pair.')
         exchange._API = exchange.Bittrex({'key': '', 'secret': ''})
-    ticker = exchange.get_ticker_history(pair)
+        tickers[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
-    dataframe = analyze.analyze_ticker(ticker)
+    else:
+        tickers = optimize.load_data(args.datadir, pairs=pairs,
+                                     ticker_interval=args.ticker_interval,
+                                     refresh_pairs=False,
+                                     timerange=timerange)
+    dataframes = optimize.tickerdata_to_dataframe(tickers)
+    dataframe = dataframes[pair]
+    dataframe = analyze.populate_buy_trend(dataframe)
+    dataframe = analyze.populate_sell_trend(dataframe)
+
+    dates = misc.datesarray_to_datetimearray(dataframe['date'])
 
     dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close']
     dataframe.loc[dataframe['sell'] == 1, 'sell_price'] = dataframe['close']
@@ -40,27 +54,30 @@ def plot_analyzed_dataframe(args):
     # Two subplots sharing x axis
     fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True)
     fig.suptitle(pair, fontsize=14, fontweight='bold')
-    ax1.plot(dataframe.index.values, dataframe['close'], label='close')
+    ax1.plot(dates, dataframe['close'], label='close')
-    # ax1.plot(dataframe.index.values, dataframe['sell'], 'ro', label='sell')
+    # ax1.plot(dates, dataframe['sell'], 'ro', label='sell')
-    ax1.plot(dataframe.index.values, dataframe['sma'], '--', label='SMA')
+    ax1.plot(dates, dataframe['sma'], '--', label='SMA')
-    ax1.plot(dataframe.index.values, dataframe['tema'], ':', label='TEMA')
+    ax1.plot(dates, dataframe['tema'], ':', label='TEMA')
-    ax1.plot(dataframe.index.values, dataframe['blower'], '-.', label='BB low')
+    ax1.plot(dates, dataframe['blower'], '-.', label='BB low')
-    ax1.plot(dataframe.index.values, dataframe['buy_price'], 'bo', label='buy')
+    ax1.plot(dates, dataframe['buy_price'], 'bo', label='buy')
     ax1.legend()
 
-    ax2.plot(dataframe.index.values, dataframe['adx'], label='ADX')
+    ax2.plot(dates, dataframe['adx'], label='ADX')
-    ax2.plot(dataframe.index.values, dataframe['mfi'], label='MFI')
+    ax2.plot(dates, dataframe['mfi'], label='MFI')
-    # ax2.plot(dataframe.index.values, [25] * len(dataframe.index.values))
+    # ax2.plot(dates, [25] * len(dataframe.index.values))
     ax2.legend()
 
-    ax3.plot(dataframe.index.values, dataframe['fastk'], label='k')
+    ax3.plot(dates, dataframe['fastk'], label='k')
-    ax3.plot(dataframe.index.values, dataframe['fastd'], label='d')
+    ax3.plot(dates, dataframe['fastd'], label='d')
-    ax3.plot(dataframe.index.values, [20] * len(dataframe.index.values))
+    ax3.plot(dates, [20] * len(dataframe.index.values))
     ax3.legend()
+    xfmt = mdates.DateFormatter('%d-%m-%y %H:%M')  # Dont let matplotlib autoformat date
+    ax3.xaxis.set_major_formatter(xfmt)
 
     # Fine-tune figure; make subplots close to each other and hide x ticks for
     # all but bottom plot.
     fig.subplots_adjust(hspace=0)
+    fig.autofmt_xdate()  # Rotate the dates
     plt.setp([a.get_xticklabels() for a in fig.axes[:-1]], visible=False)
     plt.show()
 

scripts/plot_profit.py

@@ -4,24 +4,19 @@ import sys
 import argparse
 import json
 import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
 import numpy as np
 
 import freqtrade.optimize as optimize
 import freqtrade.misc as misc
 import freqtrade.exchange as exchange
-import freqtrade.analyze  as analyze
 
 
-def plot_parse_args(args ):
+def plot_parse_args(args):
-    parser = misc.common_args_parser('Graph utility')
+    parser = misc.common_args_parser('Graph profits')
     # FIX: perhaps delete those backtesting options that are not feasible (shows up in -h)
     misc.backtesting_options(parser)
-    parser.add_argument(
+    misc.scripts_options(parser)
-        '-p', '--pair',
-        help = 'Show profits for only this pairs. Pairs are comma-separated.',
-        dest = 'pair',
-        default = None
-    )
     return parser.parse_args(args)
 
 
@@ -85,23 +80,14 @@ def plot_profit(args) -> None:
                                  timerange=timerange)
     dataframes = optimize.preprocess(tickers)
 
+    # NOTE: the dataframes are of unequal length,
+    # 'dates' is an merged date array of them all.
+
+    dates = misc.common_datearray(dataframes)
+    max_x = dates.size
+
     # Make an average close price of all the pairs that was involved.
     # this could be useful to gauge the overall market trend
-
-    # FIX: since the dataframes are of unequal length,
-    # andor has different dates, we need to merge them
-    # But we dont have the date information in the
-    # backtesting results, this is needed to match the dates
-    # For now, assume the dataframes are aligned.
-    max_x = 0
-    for pair, pair_data in dataframes.items():
-        n = len(pair_data['close'])
-        max_x = max(max_x, n)
-    #    if max_x != n:
-    #        raise Exception('Please rerun script. Input data has different lengths %s'
-    #                         %('Different pair length: %s <=> %s' %(max_x, n)))
-    print('max_x: %s' %(max_x))
-
     # We are essentially saying:
     #  array <- sum dataframes[*]['close'] / num_items dataframes
     #  FIX: there should be some onliner numpy/panda for this
@@ -131,8 +117,9 @@ def plot_profit(args) -> None:
 
     fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True)
     fig.suptitle('total profit')
-    ax1.plot(avgclose, label='avgclose')
+
-    ax2.plot(pg, label='profit')
+    ax1.plot(dates, avgclose, label='avgclose')
+    ax2.plot(dates, pg, label='profit')
     ax1.legend(loc='upper left')
     ax2.legend(loc='upper left')
 
@@ -142,15 +129,15 @@ def plot_profit(args) -> None:
     # In third graph, we plot each profit separately
     for pair in pairs:
         pg = make_profit_array(data, max_x, pair)
-        ax3.plot(pg, label=pair)
+        ax3.plot(dates, pg, label=pair)
     ax3.legend(loc='upper left')
     # black background to easier see multiple colors
     ax3.set_facecolor('black')
+    xfmt = mdates.DateFormatter('%d-%m-%y %H:%M')  # Dont let matplotlib autoformat date
+    ax3.xaxis.set_major_formatter(xfmt)
 
-    # Fine-tune figure; make subplots close to each other and hide x ticks for
-    # all but bottom plot.
     fig.subplots_adjust(hspace=0)
-    plt.setp([a.get_xticklabels() for a in fig.axes[:-1]], visible=False)
+    fig.autofmt_xdate()  # Rotate the dates
     plt.show()