Merge pull request #1513 from Axel-CH/feature/plot_df_refactoring_multiple_pairs

Feature/plot df refactoring multiple pairs

docs/plotting.md

@@ -15,7 +15,7 @@ At least version 2.3.0 is required.
 Usage for the price plotter:
 
 ```
-script/plot_dataframe.py [-h] [-p pair] [--live]
+script/plot_dataframe.py [-h] [-p pairs] [--live]
 ```
 
 Example
@@ -23,11 +23,16 @@ Example
 python scripts/plot_dataframe.py -p BTC/ETH
 ```
 
-The `-p` pair argument, can be used to specify what
+The `-p` pairs argument, can be used to specify
-pair you would like to plot.
+pairs you would like to plot.
 
 **Advanced use**
 
+To plot multiple pairs, separate them with a comma:
+```
+python scripts/plot_dataframe.py -p BTC/ETH,XRP/ETH
+```
+
 To plot the current live price use the `--live` flag:
 ```
 python scripts/plot_dataframe.py -p BTC/ETH --live

freqtrade/arguments.py

@@ -352,9 +352,9 @@ class Arguments(object):
         Parses given arguments for scripts.
         """
         self.parser.add_argument(
-            '-p', '--pair',
+            '-p', '--pairs',
             help='Show profits for only this pairs. Pairs are comma-separated.',
-            dest='pair',
+            dest='pairs',
             default=None
         )
 

freqtrade/tests/test_arguments.py

@@ -47,7 +47,7 @@ def test_scripts_options() -> None:
     arguments = Arguments(['-p', 'ETH/BTC'], '')
     arguments.scripts_options()
     args = arguments.get_parsed_arg()
-    assert args.pair == 'ETH/BTC'
+    assert args.pairs == 'ETH/BTC'
 
 
 def test_parse_args_version() -> None:

scripts/plot_dataframe.py

@@ -1,18 +1,18 @@
 #!/usr/bin/env python3
 """
-Script to display when the bot will buy a specific pair
+Script to display when the bot will buy on specific pair(s)
 
 Mandatory Cli parameters:
--p / --pair: pair to examine
+-p / --pairs: pair(s) to examine
 
 Option but recommended
 -s / --strategy: strategy to use
 
 
 Optional Cli parameters
--d / --datadir: path to pair backtest data
+-d / --datadir: path to pair(s) backtest data
 --timerange: specify what timerange of data to use.
--l / --live: Live, to download the latest ticker for the pair
+-l / --live: Live, to download the latest ticker for the pair(s)
 -db / --db-url: Show trades stored in database
 
 
@@ -21,8 +21,8 @@ Row 1: sma, ema3, ema5, ema10, ema50
 Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk
 
 Example of usage:
-> python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3
+> python3 scripts/plot_dataframe.py --pairs BTC/EUR,XRP/BTC -d user_data/data/
---indicators2 fastk,fastd
+  --indicators1 sma,ema3 --indicators2 fastk,fastd
 """
 import json
 import logging
@@ -65,7 +65,8 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram
                                 t.open_date.replace(tzinfo=timeZone),
                                 t.close_date.replace(tzinfo=timeZone) if t.close_date else None,
                                 t.open_rate, t.close_rate,
-                                t.close_date.timestamp() - t.open_date.timestamp() if t.close_date else None)
+                                t.close_date.timestamp() - t.open_date.timestamp()
+                                if t.close_date else None)
                                for t in Trade.query.filter(Trade.pair.is_(pair)).all()],
                               columns=columns)
 
@@ -74,52 +75,66 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram
         # must align with columns in backtest.py
         columns = ["pair", "profit", "opents", "closets", "index", "duration",
                    "open_rate", "close_rate", "open_at_end", "sell_reason"]
-        with file.open() as f:
+        if file.exists():
-            data = json.load(f)
+            with file.open() as f:
-            trades = pd.DataFrame(data, columns=columns)
+                data = json.load(f)
-        trades = trades.loc[trades["pair"] == pair]
+                trades = pd.DataFrame(data, columns=columns)
-        if timerange:
+            trades = trades.loc[trades["pair"] == pair]
-            if timerange.starttype == 'date':
+            if timerange:
-                trades = trades.loc[trades["opents"] >= timerange.startts]
+                if timerange.starttype == 'date':
-            if timerange.stoptype == 'date':
+                    trades = trades.loc[trades["opents"] >= timerange.startts]
-                trades = trades.loc[trades["opents"] <= timerange.stopts]
+                if timerange.stoptype == 'date':
+                    trades = trades.loc[trades["opents"] <= timerange.stopts]
+
+            trades['opents'] = pd.to_datetime(
+                                            trades['opents'],
+                                            unit='s',
+                                            utc=True,
+                                            infer_datetime_format=True)
+            trades['closets'] = pd.to_datetime(
+                                            trades['closets'],
+                                            unit='s',
+                                            utc=True,
+                                            infer_datetime_format=True)
+        else:
+            trades = pd.DataFrame([], columns=columns)
 
-        trades['opents'] = pd.to_datetime(trades['opents'],
-                                          unit='s',
-                                          utc=True,
-                                          infer_datetime_format=True)
-        trades['closets'] = pd.to_datetime(trades['closets'],
-                                           unit='s',
-                                           utc=True,
-                                           infer_datetime_format=True)
     return trades
 
 
-def plot_analyzed_dataframe(args: Namespace) -> None:
+def generate_plot_file(fig, pair, tick_interval, is_last) -> None:
     """
-    Calls analyze() and plots the returned dataframe
+    Generate a plot html file from pre populated fig plotly object
     :return: None
     """
+    logger.info('Generate plot file for %s', pair)
+
+    pair_name = pair.replace("/", "_")
+    file_name = 'freqtrade-plot-' + pair_name + '-' + tick_interval + '.html'
+
+    Path("user_data/plots").mkdir(parents=True, exist_ok=True)
+
+    plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), auto_open=False)
+    if is_last:
+        plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False)
+
+
+def get_trading_env(args: Namespace):
+    """
+    Initalize freqtrade Exchange and Strategy, split pairs recieved in parameter
+    :return: Strategy
+    """
     global _CONF
 
     # Load the configuration
     _CONF.update(setup_configuration(args))
-
     print(_CONF)
-    # Set the pair to audit
-    pair = args.pair
 
-    if pair is None:
+    pairs = args.pairs.split(',')
-        logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC')
+    if pairs is None:
+        logger.critical('Parameter --pairs mandatory;. E.g --pairs ETH/BTC,XRP/BTC')
         exit()
 
-    if '/' not in pair:
-        logger.critical('--pair format must be XXX/YYY')
-        exit()
-
-    # Set timerange to use
-    timerange = Arguments.parse_timerange(args.timerange)
-
     # Load the strategy
     try:
         strategy = StrategyResolver(_CONF).strategy
@@ -131,61 +146,84 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
         )
         exit()
 
-    # Set the ticker to use
+    return [strategy, exchange, pairs]
-    tick_interval = strategy.ticker_interval
+
+
+def get_tickers_data(strategy, exchange, pairs: List[str], args):
+    """
+    Get tickers data for each pairs on live or local, option defined in args
+    :return: dictinnary of tickers. output format: {'pair': tickersdata}
+    """
+
+    tick_interval = strategy.ticker_interval
+    timerange = Arguments.parse_timerange(args.timerange)
 
-    # Load pair tickers
     tickers = {}
     if args.live:
-        logger.info('Downloading pair.')
+        logger.info('Downloading pairs.')
-        exchange.refresh_tickers([pair], tick_interval)
+        exchange.refresh_tickers(pairs, tick_interval)
-        tickers[pair] = exchange.klines(pair)
+        for pair in pairs:
+            tickers[pair] = exchange.klines(pair)
     else:
         tickers = history.load_data(
             datadir=Path(_CONF.get("datadir")),
-            pairs=[pair],
+            pairs=pairs,
             ticker_interval=tick_interval,
             refresh_pairs=_CONF.get('refresh_pairs', False),
             timerange=timerange,
             exchange=Exchange(_CONF)
         )
 
-        # No ticker found, or impossible to download
+    # No ticker found, impossible to download, len mismatch
-        if tickers == {}:
+    for pair, data in tickers.copy().items():
-            exit()
+        logger.debug("checking tickers data of pair: %s", pair)
+        logger.debug("data.empty: %s", data.empty)
+        logger.debug("len(data): %s", len(data))
+        if data.empty:
+            del tickers[pair]
+            logger.info(
+                'An issue occured while retreiving datas of %s pair, please retry '
+                'using -l option for live or --refresh-pairs-cached', pair)
+    return tickers
 
-    # Get trades already made from the DB
+
-    trades = load_trades(args, pair, timerange)
+def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame:
+    """
+    Get tickers then Populate strategy indicators and signals, then return the full dataframe
+    :return: the DataFrame of a pair
+    """
 
     dataframes = strategy.tickerdata_to_dataframe(tickers)
-
     dataframe = dataframes[pair]
     dataframe = strategy.advise_buy(dataframe, {'pair': pair})
     dataframe = strategy.advise_sell(dataframe, {'pair': pair})
 
-    if len(dataframe.index) > args.plot_limit:
+    return dataframe
-        logger.warning('Ticker contained more than %s candles as defined '
-                       'with --plot-limit, clipping.', args.plot_limit)
-    dataframe = dataframe.tail(args.plot_limit)
 
+
+def extract_trades_of_period(dataframe, trades) -> pd.DataFrame:
+    """
+    Compare trades and backtested pair DataFrames to get trades performed on backtested period
+    :return: the DataFrame of a trades of period
+    """
     trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']]
-    fig = generate_graph(
+    return trades
-        pair=pair,
-        trades=trades,
-        data=dataframe,
-        args=args
-    )
-
-    plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')))
 
 
-def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots:
+def generate_graph(
+                    pair: str,
+                    trades: pd.DataFrame,
+                    data: pd.DataFrame,
+                    indicators1: str,
+                    indicators2: str
+                ) -> tools.make_subplots:
     """
     Generate the graph from the data generated by Backtesting or from DB
     :param pair: Pair to Display on the graph
     :param trades: All trades created
     :param data: Dataframe
-    :param args: sys.argv that contrains the two params indicators1, and indicators2
+    :indicators1: String Main plot indicators
+    :indicators2: String Sub plot indicators
     :return: None
     """
 
@@ -201,6 +239,7 @@ def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tool
     fig['layout']['yaxis1'].update(title='Price')
     fig['layout']['yaxis2'].update(title='Volume')
     fig['layout']['yaxis3'].update(title='Other')
+    fig['layout']['xaxis']['rangeslider'].update(visible=False)
 
     # Common information
     candles = go.Candlestick(
@@ -285,7 +324,7 @@ def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tool
         fig.append_trace(bb_lower, 1, 1)
         fig.append_trace(bb_upper, 1, 1)
 
-    fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data)
+    fig = generate_row(fig=fig, row=1, raw_indicators=indicators1, data=data)
     fig.append_trace(buys, 1, 1)
     fig.append_trace(sells, 1, 1)
     fig.append_trace(trade_buys, 1, 1)
@@ -300,7 +339,7 @@ def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tool
     fig.append_trace(volume, 2, 1)
 
     # Row 3
-    fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data)
+    fig = generate_row(fig=fig, row=3, raw_indicators=indicators2, data=data)
 
     return fig
 
@@ -349,7 +388,7 @@ def plot_parse_args(args: List[str]) -> Namespace:
         help='Set indicators from your strategy you want in the third row of the graph. Separate '
              'them with a coma. E.g: fastd,fastk (default: %(default)s)',
         type=str,
-        default='macd',
+        default='macd,macdsignal',
         dest='indicators2',
     )
     arguments.parser.add_argument(
@@ -366,15 +405,58 @@ def plot_parse_args(args: List[str]) -> Namespace:
     return arguments.parse_args()
 
 
+def analyse_and_plot_pairs(args: Namespace):
+    """
+    From arguments provided in cli:
+    -Initialise backtest env
+    -Get tickers data
+    -Generate Dafaframes populated with indicators and signals
+    -Load trades excecuted on same periods
+    -Generate Plotly plot objects
+    -Generate plot files
+    :return: None
+    """
+    strategy, exchange, pairs = get_trading_env(args)
+    # Set timerange to use
+    timerange = Arguments.parse_timerange(args.timerange)
+    tick_interval = strategy.ticker_interval
+
+    tickers = get_tickers_data(strategy, exchange, pairs, args)
+    pair_counter = 0
+    for pair, data in tickers.items():
+        pair_counter += 1
+        logger.info("analyse pair %s", pair)
+        tickers = {}
+        tickers[pair] = data
+        dataframe = generate_dataframe(strategy, tickers, pair)
+
+        trades = load_trades(args, pair, timerange)
+        trades = extract_trades_of_period(dataframe, trades)
+
+        fig = generate_graph(
+            pair=pair,
+            trades=trades,
+            data=dataframe,
+            indicators1=args.indicators1,
+            indicators2=args.indicators2
+        )
+
+        is_last = (False, True)[pair_counter == len(tickers)]
+        generate_plot_file(fig, pair, tick_interval, is_last)
+
+    logger.info('End of ploting process %s plots generated', pair_counter)
+
+
 def main(sysargv: List[str]) -> None:
     """
     This function will initiate the bot and start the trading loop.
     :return: None
     """
     logger.info('Starting Plot Dataframe')
-    plot_analyzed_dataframe(
+    analyse_and_plot_pairs(
         plot_parse_args(sysargv)
     )
+    exit()
 
 
 if __name__ == '__main__':

scripts/plot_profit.py

@@ -107,8 +107,8 @@ def plot_profit(args: Namespace) -> None:
         exit(1)
 
     # Take pairs from the cli otherwise switch to the pair in the config file
-    if args.pair:
+    if args.pairs:
-        filter_pairs = args.pair
+        filter_pairs = args.pairs
         filter_pairs = filter_pairs.split(',')
     else:
         filter_pairs = config['exchange']['pair_whitelist']