Merge pull request #416 from kryofly/plot_profit

Plot profit

docs/plotting.md

@@ -17,6 +17,19 @@ python script/plot_dataframe.py -p BTC_ETH,BTC_LTC
 The -p pair argument, can be used to specify what
 pair you would like to plot.
 
+**Advanced use**
+
+To plot the current live price use the --live flag:
+```
+python scripts/plot_dataframe.py -p BTC_ETH --live
+```
+
+To plot a timerange (to zoom in):
+```
+python scripts/plot_dataframe.py -p BTC_ETH --timerange=100-200
+```
+Timerange doesn't work with live data.
+
 
 ## Plot profit
 
@@ -46,3 +59,11 @@ Example
 ```
 python python scripts/plot_profit.py --datadir ../freqtrade/freqtrade/tests/testdata-20171221/ -p BTC_LTC
 ```
+
+**When it goes wrong**
+
+*** Linux: Can't display**
+
+If you are inside an python environment, you might want to set the
+DISPLAY variable as so:
+$ DISPLAY=:0 python scripts/plot_dataframe.py

freqtrade/misc.py

@@ -5,8 +5,10 @@ import logging
 import time
 import os
 import re
+from datetime import datetime
 from typing import Any, Callable, Dict, List
 
+import numpy as np
 from jsonschema import Draft4Validator, validate
 from jsonschema.exceptions import ValidationError, best_match
 from wrapt import synchronized
@@ -16,11 +18,6 @@ from freqtrade import __version__
 logger = logging.getLogger(__name__)
 
 
-def file_dump_json(filename, data):
-    with open(filename, 'w') as fp:
-        json.dump(data, fp)
-
-
 class State(enum.Enum):
     RUNNING = 0
     STOPPED = 1
@@ -30,6 +27,44 @@ class State(enum.Enum):
 _STATE = State.STOPPED
 
 
+############################################
+# Used by scripts                          #
+# Matplotlib doesn't support ::datetime64, #
+# so we need to convert it into ::datetime #
+############################################
+
+def datesarray_to_datetimearray(dates):
+    """
+    Convert an pandas-array of timestamps into
+    An numpy-array of datetimes
+    :return: numpy-array of datetime
+    """
+    times = []
+    dates = dates.astype(datetime)
+    for i in range(0, dates.size):
+        date = dates[i].to_pydatetime()
+        times.append(date)
+    return np.array(times)
+
+
+def common_datearray(dfs):
+    alldates = {}
+    for pair, pair_data in dfs.items():
+        dates = datesarray_to_datetimearray(pair_data['date'])
+        for date in dates:
+            alldates[date] = 1
+    lst = []
+    for date, _ in alldates.items():
+        lst.append(date)
+    arr = np.array(lst)
+    return np.sort(arr, axis=0)
+
+
+def file_dump_json(filename, data):
+    with open(filename, 'w') as fp:
+        json.dump(data, fp)
+
+
 @synchronized
 def update_state(state: State) -> None:
     """
@@ -163,6 +198,15 @@ def parse_args(args: List[str], description: str):
     return parser.parse_args(args)
 
 
+def scripts_options(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        '-p', '--pair',
+        help='Show profits for only this pairs. Pairs are comma-separated.',
+        dest='pair',
+        default=None
+    )
+
+
 def backtesting_options(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         '-l', '--live',

scripts/plot_dataframe.py

@@ -1,34 +1,32 @@
 #!/usr/bin/env python3
 
 import sys
-import talib.abstract as ta
+import logging
-import freqtrade.vendor.qtpylib.indicators as qtpylib
+import argparse
+
+import matplotlib
+# matplotlib.use("Qt5Agg")
+import matplotlib.dates as mdates
+import matplotlib.pyplot as plt
 from pandas import DataFrame
+import talib.abstract as ta
+
+import freqtrade.vendor.qtpylib.indicators as qtpylib
 from freqtrade import exchange, analyze
 from freqtrade.misc import common_args_parser
 from freqtrade.strategy.strategy import Strategy
-import matplotlib.pyplot as plt
+import freqtrade.misc as misc
-import matplotlib  # Install PYQT5 manually if you want to test this helper function
+import freqtrade.optimize as optimize
+import freqtrade.analyze as analyze
 
-matplotlib.use("Qt5Agg")
+
+logger = logging.getLogger(__name__)
 
 
 def plot_parse_args(args):
-    parser = common_args_parser(description='Graph utility')
+    parser = misc.common_args_parser('Graph dataframe')
-    parser.add_argument(
+    misc.backtesting_options(parser)
-        '-p', '--pair',
+    misc.scripts_options(parser)
-        help='What currency pair',
-        dest='pair',
-        default='BTC_ETH',
-        type=str,
-    )
-    parser.add_argument(
-        '-i', '--interval',
-        help='what interval to use',
-        dest='interval',
-        default=5,
-        type=int,
-    )
     return parser.parse_args(args)
 
 
@@ -38,80 +36,65 @@ def plot_analyzed_dataframe(args) -> None:
     :param pair: pair as str
     :return: None
     """
+    pair = args.pair
+    pairs = [pair]
+    timerange = misc.parse_timerange(args.timerange)
+
     # Init strategy
     strategy = Strategy()
     strategy.init({'strategy': args.strategy})
+    tick_interval = strategy.ticker_interval
 
+    tickers = {}
+    if args.live:
+        logger.info('Downloading pair.')
         # Init Bittrex to use public API
         exchange._API = exchange.Bittrex({'key': '', 'secret': ''})
-    ticker = exchange.get_ticker_history(args.pair, args.interval)
+        tickers[pair] = exchange.get_ticker_history(pair, tick_interval)
-    dataframe = analyze.analyze_ticker(ticker)
+    else:
-
+        tickers = optimize.load_data(args.datadir, pairs=pairs,
-    dataframe = populate_indicator(dataframe)
+                                     ticker_interval=tick_interval,
+                                     refresh_pairs=False,
+                                     timerange=timerange)
+    dataframes = optimize.tickerdata_to_dataframe(tickers)
+    dataframe = dataframes[pair]
+    dataframe = analyze.populate_buy_trend(dataframe)
+    dataframe = analyze.populate_sell_trend(dataframe)
+    dates = misc.datesarray_to_datetimearray(dataframe['date'])
 
     # Two subplots sharing x axis
     fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True)
-    fig.suptitle(args.pair + " " + str(args.interval), fontsize=14, fontweight='bold')
+    fig.suptitle(pair + " " +     str(tick_interval), fontsize=14, fontweight='bold')
-    ax1.plot(dataframe.index.values, dataframe['close'], label='close')
+
-    # ax1.plot(dataframe.index.values, dataframe['sell'], 'ro', label='sell')
+    ax1.plot(dates, dataframe['close'], label='close')
-    ax1.plot(dataframe.index.values, dataframe['sma'], '--', label='SMA')
+    # ax1.plot(dates, dataframe['sell'], 'ro', label='sell')
-    ax1.plot(dataframe.index.values, dataframe['tema'], ':', label='TEMA')
+    ax1.plot(dates, dataframe['sma'], '--', label='SMA')
-    ax1.plot(dataframe.index.values, dataframe['bb_lowerband'], '-.', label='BB low')
+    ax1.plot(dates, dataframe['tema'], ':', label='TEMA')
-    ax1.plot(dataframe.index.values, dataframe['buy_price'], 'bo', label='buy')
+    ax1.plot(dates, dataframe['blower'], '-.', label='BB low')
+    ax1.plot(dates, dataframe['close'] * dataframe['buy'], 'bo', label='buy')
+    ax1.plot(dates, dataframe['close'] * dataframe['sell'], 'ro', label='sell')
+
     ax1.legend()
 
-    ax2.plot(dataframe.index.values, dataframe['adx'], label='ADX')
+    ax2.plot(dates, dataframe['adx'], label='ADX')
-    ax2.plot(dataframe.index.values, dataframe['mfi'], label='MFI')
+    ax2.plot(dates, dataframe['mfi'], label='MFI')
-    # ax2.plot(dataframe.index.values, [25] * len(dataframe.index.values))
+    # ax2.plot(dates, [25] * len(dataframe.index.values))
     ax2.legend()
 
-    ax3.plot(dataframe.index.values, dataframe['fastk'], label='k')
+    ax3.plot(dates, dataframe['fastk'], label='k')
-    ax3.plot(dataframe.index.values, dataframe['fastd'], label='d')
+    ax3.plot(dates, dataframe['fastd'], label='d')
-    ax3.plot(dataframe.index.values, [20] * len(dataframe.index.values))
+    ax3.plot(dates, [20] * len(dataframe.index.values))
     ax3.legend()
+    xfmt = mdates.DateFormatter('%d-%m-%y %H:%M')  # Dont let matplotlib autoformat date
+    ax3.xaxis.set_major_formatter(xfmt)
 
     # Fine-tune figure; make subplots close to each other and hide x ticks for
     # all but bottom plot.
     fig.subplots_adjust(hspace=0)
+    fig.autofmt_xdate()  # Rotate the dates
     plt.setp([a.get_xticklabels() for a in fig.axes[:-1]], visible=False)
     plt.show()
 
-
-def populate_indicator(dataframe: DataFrame) -> DataFrame:
-
-    dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close']
-    dataframe.loc[dataframe['sell'] == 1, 'sell_price'] = dataframe['close']
-
-    # ADX
-    if 'adx' not in dataframe:
-        dataframe['adx'] = ta.ADX(dataframe)
-
-    # Bollinger bands
-    if 'bb_lowerband' not in dataframe:
-        bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
-        dataframe['bb_lowerband'] = bollinger['lower']
-
-    # Stoch fast
-    if 'fastd' not in dataframe or 'fastk' not in dataframe:
-        stoch_fast = ta.STOCHF(dataframe)
-        dataframe['fastd'] = stoch_fast['fastd']
-        dataframe['fastk'] = stoch_fast['fastk']
-
-    # MFI
-    if 'mfi' not in dataframe:
-        dataframe['mfi'] = ta.MFI(dataframe)
-
-    # SMA - Simple Moving Average
-    if 'sma' not in dataframe:
-        dataframe['sma'] = ta.SMA(dataframe, timeperiod=40)
-
-    # TEMA - Triple Exponential Moving Average
-    if 'tema' not in dataframe:
-        dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9)
-
-    return dataframe
-
-
 if __name__ == '__main__':
     args = plot_parse_args(sys.argv[1:])
     plot_analyzed_dataframe(args)

scripts/plot_profit.py

@@ -3,23 +3,20 @@
 import sys
 import json
 import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
 import numpy as np
 
 import freqtrade.optimize as optimize
 import freqtrade.misc as misc
+import freqtrade.exchange as exchange
 from freqtrade.strategy.strategy import Strategy
 
 
 def plot_parse_args(args):
-    parser = misc.common_args_parser('Graph utility')
+    parser = misc.common_args_parser('Graph profits')
     # FIX: perhaps delete those backtesting options that are not feasible (shows up in -h)
     misc.backtesting_options(parser)
-    parser.add_argument(
+    misc.scripts_options(parser)
-        '-p', '--pair',
-        help='Show profits for only this pairs. Pairs are comma-separated.',
-        dest='pair',
-        default=None
-    )
     return parser.parse_args(args)
 
 
@@ -39,7 +36,9 @@ def make_profit_array(data, px, filter_pairs=[]):
         profit = trade[1]
         tim = trade[4]
         dur = trade[5]
-        pg[tim+dur-1] += profit
+        ix = tim + dur - 1
+        if ix < px:
+            pg[ix] += profit
 
     # rewrite the pg array to go from
     # total profits at each timeframe
@@ -81,28 +80,21 @@ def plot_profit(args) -> None:
         pairs = list(set(pairs) & set(filter_pairs))
         print('Filter, keep pairs %s' % pairs)
 
+    timerange = misc.parse_timerange(args.timerange)
     tickers = optimize.load_data(args.datadir, pairs=pairs,
                                  ticker_interval=strategy.ticker_interval,
-                                 refresh_pairs=False)
+                                 refresh_pairs=False,
+                                 timerange=timerange)
     dataframes = optimize.preprocess(tickers)
 
+    # NOTE: the dataframes are of unequal length,
+    # 'dates' is an merged date array of them all.
+
+    dates = misc.common_datearray(dataframes)
+    max_x = dates.size
+
     # Make an average close price of all the pairs that was involved.
     # this could be useful to gauge the overall market trend
-
-    # FIX: since the dataframes are of unequal length,
-    # andor has different dates, we need to merge them
-    # But we dont have the date information in the
-    # backtesting results, this is needed to match the dates
-    # For now, assume the dataframes are aligned.
-    max_x = 0
-    for pair, pair_data in dataframes.items():
-        n = len(pair_data['close'])
-        max_x = max(max_x, n)
-    #    if max_x != n:
-    #        raise Exception('Please rerun script. Input data has different lengths %s'
-    #                         %('Different pair length: %s <=> %s' %(max_x, n)))
-    print('max_x: %s' % (max_x))
-
     # We are essentially saying:
     #  array <- sum dataframes[*]['close'] / num_items dataframes
     #  FIX: there should be some onliner numpy/panda for this
@@ -132,8 +124,9 @@ def plot_profit(args) -> None:
 
     fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True)
     fig.suptitle('total profit')
-    ax1.plot(avgclose, label='avgclose')
+
-    ax2.plot(pg, label='profit')
+    ax1.plot(dates, avgclose, label='avgclose')
+    ax2.plot(dates, pg, label='profit')
     ax1.legend(loc='upper left')
     ax2.legend(loc='upper left')
 
@@ -143,15 +136,15 @@ def plot_profit(args) -> None:
     # In third graph, we plot each profit separately
     for pair in pairs:
         pg = make_profit_array(data, max_x, pair)
-        ax3.plot(pg, label=pair)
+        ax3.plot(dates, pg, label=pair)
     ax3.legend(loc='upper left')
     # black background to easier see multiple colors
     ax3.set_facecolor('black')
+    xfmt = mdates.DateFormatter('%d-%m-%y %H:%M')  # Dont let matplotlib autoformat date
+    ax3.xaxis.set_major_formatter(xfmt)
 
-    # Fine-tune figure; make subplots close to each other and hide x ticks for
-    # all but bottom plot.
     fig.subplots_adjust(hspace=0)
-    plt.setp([a.get_xticklabels() for a in fig.axes[:-1]], visible=False)
+    fig.autofmt_xdate()  # Rotate the dates
     plt.show()