From 829da096e23072269562da4e57a42e199a292161 Mon Sep 17 00:00:00 2001
From: kryofly <kryofly@space>
Date: Fri, 12 Jan 2018 11:49:50 +0100
Subject: [PATCH 1/5] plotting docs

---
 docs/plotting.md          | 18 ++++++++++++++++++
 scripts/plot_dataframe.py |  2 +-
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 docs/plotting.md

diff --git a/docs/plotting.md b/docs/plotting.md
new file mode 100644
index 000000000..56b44400c
--- /dev/null
+++ b/docs/plotting.md
@@ -0,0 +1,18 @@
+# Plotting
+This page explains how to plot prices, indicator, profits.
+
+## Table of Contents
+- [Plot price and indicators](#plot-price-and-indicators)
+
+## Plot price and indicators
+Usage for the price plotter:
+script/plot_dataframe.py [-h] [-p pair]
+
+Example
+```
+python script/plot_dataframe.py -p BTC_ETH
+```
+
+The -p pair argument, can be used to specify what
+pair you would like to plot.
+
diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py
index f07033637..e9bf65f47 100755
--- a/scripts/plot_dataframe.py
+++ b/scripts/plot_dataframe.py
@@ -21,7 +21,7 @@ def plot_parse_args(args ):
     return parser.parse_args(args)
 
 
-def plot_analyzed_dataframe(args) -> None:
+def plot_analyzed_dataframe(args):
     """
     Calls analyze() and plots the returned dataframe
     :param pair: pair as str

From 98cf98693468c57335ad38fa9e5b6b9ac02ec66f Mon Sep 17 00:00:00 2001
From: kryofly <kryofly@space>
Date: Fri, 12 Jan 2018 10:52:44 +0100
Subject: [PATCH 2/5] misc options parsing split up

---
 freqtrade/misc.py | 122 ++++++++++++++++++++++++----------------------
 1 file changed, 65 insertions(+), 57 deletions(-)

diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 3d70f6b25..92fe39ef2 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -110,6 +110,14 @@ def common_args_parser(description: str):
         type=str,
         metavar='PATH',
     )
+    parser.add_argument(
+        '-dd', '--datadir',
+        help='path to backtest data (default freqdata/tests/testdata',
+        dest='datadir',
+        default=os.path.join('freqtrade', 'tests', 'testdata'),
+        type=str,
+        metavar='PATH',
+    )
     return parser
 
 
@@ -126,14 +134,6 @@ def parse_args(args: List[str], description: str):
         action='store_true',
         dest='dry_run_db',
     )
-    parser.add_argument(
-        '-dd', '--datadir',
-        help='path to backtest data (default freqdata/tests/testdata',
-        dest='datadir',
-        default=os.path.join('freqtrade', 'tests', 'testdata'),
-        type=str,
-        metavar='PATH',
-    )
     parser.add_argument(
         '--dynamic-whitelist',
         help='dynamically generate and update whitelist \
@@ -149,6 +149,61 @@ def parse_args(args: List[str], description: str):
     return parser.parse_args(args)
 
 
+def backtesting_options(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        '-l', '--live',
+        action='store_true',
+        dest='live',
+        help='using live data',
+    )
+    parser.add_argument(
+        '-i', '--ticker-interval',
+        help='specify ticker interval in minutes (default: 5)',
+        dest='ticker_interval',
+        default=5,
+        type=int,
+        metavar='INT',
+    )
+    parser.add_argument(
+        '--realistic-simulation',
+        help='uses max_open_trades from config to simulate real world limitations',
+        action='store_true',
+        dest='realistic_simulation',
+    )
+    parser.add_argument(
+        '-r', '--refresh-pairs-cached',
+        help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \
+              Use it if you want to run your backtesting with up-to-date data.',
+        action='store_true',
+        dest='refresh_pairs',
+    )
+
+
+def hyperopt_options(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        '-e', '--epochs',
+        help='specify number of epochs (default: 100)',
+        dest='epochs',
+        default=100,
+        type=int,
+        metavar='INT',
+    )
+    parser.add_argument(
+        '--use-mongodb',
+        help='parallelize evaluations with mongodb (requires mongod in PATH)',
+        dest='mongodb',
+        action='store_true',
+    )
+    parser.add_argument(
+        '-i', '--ticker-interval',
+        help='specify ticker interval in minutes (default: 5)',
+        dest='ticker_interval',
+        default=5,
+        type=int,
+        metavar='INT',
+    )
+
+
 def build_subcommands(parser: argparse.ArgumentParser) -> None:
     """ Builds and attaches all subcommands """
     from freqtrade.optimize import backtesting, hyperopt
@@ -158,59 +213,12 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
     # Add backtesting subcommand
     backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module')
     backtesting_cmd.set_defaults(func=backtesting.start)
-    backtesting_cmd.add_argument(
-        '-l', '--live',
-        action='store_true',
-        dest='live',
-        help='using live data',
-    )
-    backtesting_cmd.add_argument(
-        '-i', '--ticker-interval',
-        help='specify ticker interval in minutes (default: 5)',
-        dest='ticker_interval',
-        default=5,
-        type=int,
-        metavar='INT',
-    )
-    backtesting_cmd.add_argument(
-        '--realistic-simulation',
-        help='uses max_open_trades from config to simulate real world limitations',
-        action='store_true',
-        dest='realistic_simulation',
-    )
-    backtesting_cmd.add_argument(
-        '-r', '--refresh-pairs-cached',
-        help='refresh the pairs files in tests/testdata with the latest data from Bittrex. \
-              Use it if you want to run your backtesting with up-to-date data.',
-        action='store_true',
-        dest='refresh_pairs',
-    )
+    backtesting_options(backtesting_cmd)
 
     # Add hyperopt subcommand
     hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
     hyperopt_cmd.set_defaults(func=hyperopt.start)
-    hyperopt_cmd.add_argument(
-        '-e', '--epochs',
-        help='specify number of epochs (default: 100)',
-        dest='epochs',
-        default=100,
-        type=int,
-        metavar='INT',
-    )
-    hyperopt_cmd.add_argument(
-        '--use-mongodb',
-        help='parallelize evaluations with mongodb (requires mongod in PATH)',
-        dest='mongodb',
-        action='store_true',
-    )
-    hyperopt_cmd.add_argument(
-        '-i', '--ticker-interval',
-        help='specify ticker interval in minutes (default: 5)',
-        dest='ticker_interval',
-        default=5,
-        type=int,
-        metavar='INT',
-    )
+    hyperopt_options(hyperopt_cmd)
 
 
 # Required json-schema for user specified config

From d8d46890b396a17f51099e9d29860820f7f40af8 Mon Sep 17 00:00:00 2001
From: kryofly <kryofly@space>
Date: Fri, 12 Jan 2018 10:55:49 +0100
Subject: [PATCH 3/5] script: plot profit

---
 docs/plotting.md       |  30 ++++++++
 scripts/plot_profit.py | 158 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 188 insertions(+)
 create mode 100755 scripts/plot_profit.py

diff --git a/docs/plotting.md b/docs/plotting.md
index 56b44400c..f0df496ac 100644
--- a/docs/plotting.md
+++ b/docs/plotting.md
@@ -3,6 +3,7 @@ This page explains how to plot prices, indicator, profits.
 
 ## Table of Contents
 - [Plot price and indicators](#plot-price-and-indicators)
+- [Plot profit](#plot-profit)
 
 ## Plot price and indicators
 Usage for the price plotter:
@@ -16,3 +17,32 @@ python script/plot_dataframe.py -p BTC_ETH
 The -p pair argument, can be used to specify what
 pair you would like to plot.
 
+
+## Plot profit
+
+The profit plotter show a picture with three plots:
+1) Average closing price for all pairs
+2) The summarized profit made by backtesting.
+   Note that this is not the real-world profit, but
+   more of an estimate.
+3) Each pair individually profit
+
+The first graph is good to get a grip of how the overall market
+progresses.
+
+The second graph will show how you algorithm works or doesnt.
+Perhaps you want an algorithm that steadily makes small profits,
+or one that acts less seldom, but makes big swings.
+
+The third graph can be useful to spot outliers, events in pairs
+that makes profit spikes.
+
+Usage for the profit plotter:
+script/plot_profit.py [-h] [-p pair] [--datadir directory] [--ticker_interval num]
+
+The -p pair argument, can be used to plot a single pair
+
+Example
+```
+python python scripts/plot_profit.py --datadir ../freqtrade/freqtrade/tests/testdata-20171221/ -p BTC_LTC
+```
diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py
new file mode 100755
index 000000000..ad8346455
--- /dev/null
+++ b/scripts/plot_profit.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+
+import sys
+import argparse
+import json
+import matplotlib.pyplot as plt
+import numpy as np
+
+import freqtrade.optimize as optimize
+import freqtrade.misc as misc
+import freqtrade.exchange as exchange
+import freqtrade.analyze  as analyze
+
+
+def plot_parse_args(args ):
+    parser = misc.common_args_parser('Graph utility')
+    # FIX: perhaps delete those backtesting options that are not feasible
+    misc.backtesting_options(parser)
+    # TODO: Make the pair argument take a comma separated list
+    parser.add_argument(
+        '-p', '--pair',
+        help = 'Show profits for only this pair',
+        dest = 'pair',
+        default = None
+    )
+
+    return parser.parse_args(args)
+
+
+def make_profit_array(data, filter_pair):
+    xmin = 0
+    xmax = 0
+
+    #  pair       profit-%    time  duration
+    # ['BTC_XMR', 0.00537847, 5057, 1]
+    for trade in data:
+        pair = trade[0]
+        profit = trade[1]
+        x = trade[2]
+        dur = trade[3]
+        xmax = max(xmax, x + dur)
+
+    pg = np.zeros(xmax)
+
+    # Go through the trades
+    # and make an total profit
+    # array
+    for trade in data:
+        pair = trade[0]
+        if filter_pair and pair != filter_pair:
+            continue
+        profit = trade[1]
+        tim = trade[2]
+        dur = trade[3]
+        pg[tim+dur-1] += profit
+
+    # rewrite the pg array to go from
+    # total profits at each timeframe
+    # to accumulated profits
+    pa = 0
+    for x in range(0,len(pg)):
+        p = pg[x]  # Get current total percent
+        pa += p  # Add to the accumulated percent
+        pg[x] = pa  # write back to save memory
+
+    return pg
+
+
+def plot_profit(args) -> None:
+    """
+    Plots the total profit for all pairs.
+    Note, the profit calculation isn't realistic.
+    But should be somewhat proportional, and therefor useful
+    in helping out to find a good algorithm.
+    """
+
+    # We need to use the same pairs, same tick_interval
+    # and same timeperiod as used in backtesting
+    # to match the tickerdata against the profits-results
+
+    filter_pair = args.pair
+
+    config = misc.load_config(args.config)
+    pairs = config['exchange']['pair_whitelist']
+    if filter_pair:
+        print('Filtering out pair %s' % filter_pair)
+        pairs = list(filter(lambda pair: pair == filter_pair, pairs))
+
+    tickers = optimize.load_data(args.datadir, pairs=pairs,
+                                 ticker_interval=args.ticker_interval,
+                                 refresh_pairs=False)
+    dataframes = optimize.preprocess(tickers)
+
+    # Make an average close price of all the pairs that was involved.
+    # this could be useful to gauge the overall market trend
+
+    # FIX: since the dataframes are of unequal length,
+    # andor has different dates, we need to merge them
+    # But we dont have the date information in the
+    # backtesting results, this is needed to match the dates
+    # For now, assume the dataframes are aligned.
+
+    # We are essentially saying:
+    #  array <- sum dataframes[*]['close'] / num_items dataframes
+    #  FIX: there should be some onliner numpy/panda for this
+
+    first = True
+    avgclose = None
+    num = 0
+    for pair, pair_data in dataframes.items():
+      close = pair_data['close']
+      print('Pair %s has length %s' %(pair, len(close)))
+      num += 1
+      if first:
+        first = False
+        avgclose = np.copy(close)
+      else:
+        avgclose += close
+    avgclose /= num
+
+    # Load the profits results
+    # And make an profits-growth array
+
+    filename = 'backtest-result.json'
+    with open(filename) as file:
+      data = json.load(file)
+    pg = make_profit_array(data, filter_pair)
+
+    #
+    # Plot the pairs average close prices, and total profit growth
+    #
+
+    fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True)
+    fig.suptitle('total profit')
+    ax1.plot(avgclose, label='avgclose')
+    ax2.plot(pg, label='profit')
+    ax1.legend()
+    ax2.legend()
+
+    # FIX if we have one line pair in paris
+    #     then skip the plotting of the third graph,
+    #     or change what we plot
+    # In third graph, we plot each profit separately
+    for pair in pairs:
+        pg = make_profit_array(data, pair)
+        ax3.plot(pg, label=pair)
+    ax3.legend()
+
+    # Fine-tune figure; make subplots close to each other and hide x ticks for
+    # all but bottom plot.
+    fig.subplots_adjust(hspace=0)
+    plt.setp([a.get_xticklabels() for a in fig.axes[:-1]], visible=False)
+    plt.show()
+
+
+if __name__ == '__main__':
+    args = plot_parse_args(sys.argv[1:])
+    plot_profit(args)

From 167483f7771c87767501e63279e701491426c829 Mon Sep 17 00:00:00 2001
From: kryofly <kryofly@space>
Date: Fri, 12 Jan 2018 19:18:31 +0100
Subject: [PATCH 4/5] plot profit: filter multiple pairs, misc fixes

---
 docs/plotting.md       |  2 +-
 freqtrade/misc.py      |  2 +-
 scripts/plot_profit.py | 75 ++++++++++++++++++++----------------------
 3 files changed, 37 insertions(+), 42 deletions(-)

diff --git a/docs/plotting.md b/docs/plotting.md
index f0df496ac..598443e12 100644
--- a/docs/plotting.md
+++ b/docs/plotting.md
@@ -11,7 +11,7 @@ script/plot_dataframe.py [-h] [-p pair]
 
 Example
 ```
-python script/plot_dataframe.py -p BTC_ETH
+python script/plot_dataframe.py -p BTC_ETH,BTC_LTC
 ```
 
 The -p pair argument, can be used to specify what
diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 92fe39ef2..97e885e01 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -111,7 +111,7 @@ def common_args_parser(description: str):
         metavar='PATH',
     )
     parser.add_argument(
-        '-dd', '--datadir',
+        '--datadir',
         help='path to backtest data (default freqdata/tests/testdata',
         dest='datadir',
         default=os.path.join('freqtrade', 'tests', 'testdata'),
diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py
index ad8346455..a61335318 100755
--- a/scripts/plot_profit.py
+++ b/scripts/plot_profit.py
@@ -14,40 +14,27 @@ import freqtrade.analyze  as analyze
 
 def plot_parse_args(args ):
     parser = misc.common_args_parser('Graph utility')
-    # FIX: perhaps delete those backtesting options that are not feasible
+    # FIX: perhaps delete those backtesting options that are not feasible (shows up in -h)
     misc.backtesting_options(parser)
-    # TODO: Make the pair argument take a comma separated list
     parser.add_argument(
         '-p', '--pair',
-        help = 'Show profits for only this pair',
+        help = 'Show profits for only this pairs. Pairs are comma-separated.',
         dest = 'pair',
         default = None
     )
-
     return parser.parse_args(args)
 
 
-def make_profit_array(data, filter_pair):
-    xmin = 0
-    xmax = 0
-
-    #  pair       profit-%    time  duration
-    # ['BTC_XMR', 0.00537847, 5057, 1]
-    for trade in data:
-        pair = trade[0]
-        profit = trade[1]
-        x = trade[2]
-        dur = trade[3]
-        xmax = max(xmax, x + dur)
-
-    pg = np.zeros(xmax)
-
+# data:: [ pair,      profit-%,  time, duration]
+# data:: ['BTC_XMR', 0.00537847, 5057, 1]
+def make_profit_array(data, px, filter_pairs=[]):
+    pg = np.zeros(px)
     # Go through the trades
     # and make an total profit
     # array
     for trade in data:
         pair = trade[0]
-        if filter_pair and pair != filter_pair:
+        if filter_pairs and pair not in filter_pairs:
             continue
         profit = trade[1]
         tim = trade[2]
@@ -78,13 +65,14 @@ def plot_profit(args) -> None:
     # and same timeperiod as used in backtesting
     # to match the tickerdata against the profits-results
 
-    filter_pair = args.pair
+    filter_pairs = args.pair
 
     config = misc.load_config(args.config)
     pairs = config['exchange']['pair_whitelist']
-    if filter_pair:
-        print('Filtering out pair %s' % filter_pair)
-        pairs = list(filter(lambda pair: pair == filter_pair, pairs))
+    if filter_pairs:
+        filter_pairs = filter_pairs.split(',')
+        pairs = list(set(pairs) & set(filter_pairs))
+        print('Filter, keep pairs %s' % pairs)
 
     tickers = optimize.load_data(args.datadir, pairs=pairs,
                                  ticker_interval=args.ticker_interval,
@@ -99,23 +87,28 @@ def plot_profit(args) -> None:
     # But we dont have the date information in the
     # backtesting results, this is needed to match the dates
     # For now, assume the dataframes are aligned.
+    max_x = 0
+    for pair, pair_data in dataframes.items():
+        n = len(pair_data['close'])
+        max_x = max(max_x, n)
+    #    if max_x != n:
+    #        raise Exception('Please rerun script. Input data has different lengths %s'
+    #                         %('Different pair length: %s <=> %s' %(max_x, n)))
+    print('max_x: %s' %(max_x))
 
     # We are essentially saying:
     #  array <- sum dataframes[*]['close'] / num_items dataframes
     #  FIX: there should be some onliner numpy/panda for this
-
-    first = True
-    avgclose = None
+    avgclose = np.zeros(max_x)
     num = 0
     for pair, pair_data in dataframes.items():
-      close = pair_data['close']
-      print('Pair %s has length %s' %(pair, len(close)))
-      num += 1
-      if first:
-        first = False
-        avgclose = np.copy(close)
-      else:
-        avgclose += close
+        close = pair_data['close']
+        maxprice = max(close)  # Normalize price to [0,1]
+        print('Pair %s has length %s' %(pair, len(close)))
+        for x in range(0, len(close)):
+            avgclose[x] += close[x] / maxprice
+        # avgclose += close
+        num += 1
     avgclose /= num
 
     # Load the profits results
@@ -124,7 +117,7 @@ def plot_profit(args) -> None:
     filename = 'backtest-result.json'
     with open(filename) as file:
       data = json.load(file)
-    pg = make_profit_array(data, filter_pair)
+    pg = make_profit_array(data, max_x, filter_pairs)
 
     #
     # Plot the pairs average close prices, and total profit growth
@@ -134,17 +127,19 @@ def plot_profit(args) -> None:
     fig.suptitle('total profit')
     ax1.plot(avgclose, label='avgclose')
     ax2.plot(pg, label='profit')
-    ax1.legend()
-    ax2.legend()
+    ax1.legend(loc='upper left')
+    ax2.legend(loc='upper left')
 
     # FIX if we have one line pair in paris
     #     then skip the plotting of the third graph,
     #     or change what we plot
     # In third graph, we plot each profit separately
     for pair in pairs:
-        pg = make_profit_array(data, pair)
+        pg = make_profit_array(data, max_x, pair)
         ax3.plot(pg, label=pair)
-    ax3.legend()
+    ax3.legend(loc='upper left')
+    # black background to easier see multiple colors
+    ax3.set_facecolor('black')
 
     # Fine-tune figure; make subplots close to each other and hide x ticks for
     # all but bottom plot.

From 524899ccbf8629b7db97920a5386201ebd0ad76c Mon Sep 17 00:00:00 2001
From: kryofly <kryofly@space>
Date: Fri, 12 Jan 2018 22:15:50 +0100
Subject: [PATCH 5/5] plot profit: export format change

---
 scripts/plot_profit.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py
index a61335318..e8bdbee5c 100755
--- a/scripts/plot_profit.py
+++ b/scripts/plot_profit.py
@@ -25,8 +25,10 @@ def plot_parse_args(args ):
     return parser.parse_args(args)
 
 
-# data:: [ pair,      profit-%,  time, duration]
-# data:: ['BTC_XMR', 0.00537847, 5057, 1]
+# data:: [ pair,      profit-%,  enter,         exit,        time, duration]
+# data:: ['BTC_XMR', 0.00537847, '1511176800', '1511178000', 5057, 1]
+# FIX: make use of the enter/exit dates to insert the
+# profit more precisely into the pg array
 def make_profit_array(data, px, filter_pairs=[]):
     pg = np.zeros(px)
     # Go through the trades
@@ -37,8 +39,8 @@ def make_profit_array(data, px, filter_pairs=[]):
         if filter_pairs and pair not in filter_pairs:
             continue
         profit = trade[1]
-        tim = trade[2]
-        dur = trade[3]
+        tim = trade[4]
+        dur = trade[5]
         pg[tim+dur-1] += profit
 
     # rewrite the pg array to go from